From ee9dea06fbea9edb0c7c42ca9994eee058bc62d8 Mon Sep 17 00:00:00 2001 From: marcsello Date: Wed, 4 Aug 2021 15:20:29 +0200 Subject: [PATCH] tracing fixes --- cnn_classification_service/cnn_classifier.py | 48 ++++++++++---------- cnn_classification_service/magic_doer.py | 10 ++-- cnn_classification_service/main.py | 6 +-- 3 files changed, 34 insertions(+), 30 deletions(-) diff --git a/cnn_classification_service/cnn_classifier.py b/cnn_classification_service/cnn_classifier.py index 4d368a1..4ced69c 100644 --- a/cnn_classification_service/cnn_classifier.py +++ b/cnn_classification_service/cnn_classifier.py @@ -29,28 +29,32 @@ class Classifier(object): @staticmethod def create_spectrogram(wav_filename: str) -> Tuple[str, str]: - matplotlib.pyplot.interactive(False) - clip, sample_rate = librosa.load(wav_filename, sr=None) - fig = matplotlib.pyplot.figure(figsize=[0.72, 0.72]) - ax = fig.add_subplot(111) - ax.axes.get_xaxis().set_visible(False) - ax.axes.get_yaxis().set_visible(False) - ax.set_frame_on(False) - spectogram = librosa.feature.melspectrogram(y=clip, sr=sample_rate) - librosa.display.specshow(librosa.power_to_db(spectogram, ref=numpy.max)) - target_dir = tempfile.mkdtemp(dir="/dev/shm") - wav_basename = os.path.basename(wav_filename) + with opentracing.tracer.start_active_span('classifier.librosa.load'): + clip, sample_rate = librosa.load(wav_filename, sr=None) - # Change extension to jpg... mert 110% biztos vagyok benne hogy a keras nem bírná beolvasni máshogy - file_name = os.path.join(target_dir, "unknown", f"{wav_basename[:-4]}.jpg") - os.mkdir(os.path.join(target_dir, "unknown")) + with opentracing.tracer.start_active_span('classifier.Plot'): + matplotlib.pyplot.interactive(False) + fig = matplotlib.pyplot.figure(figsize=[0.72, 0.72]) + ax = fig.add_subplot(111) + ax.axes.get_xaxis().set_visible(False) + ax.axes.get_yaxis().set_visible(False) + ax.set_frame_on(False) + spectogram = librosa.feature.melspectrogram(y=clip, sr=sample_rate) + librosa.display.specshow(librosa.power_to_db(spectogram, ref=numpy.max)) - matplotlib.pyplot.savefig(file_name, dpi=400, bbox_inches='tight', pad_inches=0) - matplotlib.pyplot.close() - fig.clf() - matplotlib.pyplot.close(fig) - matplotlib.pyplot.close('all') + target_dir = tempfile.mkdtemp(dir="/dev/shm") + wav_basename = os.path.basename(wav_filename) + + # Change extension to jpg... mert 110% biztos vagyok benne hogy a keras nem bírná beolvasni máshogy + file_name = os.path.join(target_dir, "unknown", f"{wav_basename[:-4]}.jpg") + os.mkdir(os.path.join(target_dir, "unknown")) + + matplotlib.pyplot.savefig(file_name, dpi=400, bbox_inches='tight', pad_inches=0) + matplotlib.pyplot.close() + fig.clf() + matplotlib.pyplot.close(fig) + matplotlib.pyplot.close('all') return target_dir, file_name # Az unknown nélkülivel kell visszatérni @@ -85,12 +89,10 @@ class Classifier(object): return predicted_class_name, labeled_predictions def predict(self, wav_filename: str) -> Tuple[str, dict]: - span = opentracing.tracer.scope_manager.active().span - - with opentracing.tracer.start_active_span('createSpectrogram', child_of=span): + with opentracing.tracer.start_active_span('classifier.createSpectrogram'): directory, _ = self.create_spectrogram(wav_filename) - with opentracing.tracer.start_active_span('runPredictor', child_of=span): + with opentracing.tracer.start_active_span('classifier.runPredictor'): result = self._run_predictor(directory) shutil.rmtree(directory) # The image is no longer needed diff --git a/cnn_classification_service/magic_doer.py b/cnn_classification_service/magic_doer.py index 337dec2..5026537 100644 --- a/cnn_classification_service/magic_doer.py +++ b/cnn_classification_service/magic_doer.py @@ -19,15 +19,16 @@ class MagicDoer: @classmethod def run_everything(cls, parameters: dict) -> dict: - span = opentracing.tracer.scope_manager.active().span + tag = parameters['tag'] sample_file_handle, sample_file_path = tempfile.mkstemp(prefix=f"{tag}_", suffix=".wav", dir="/dev/shm") + span = opentracing.tracer.scope_manager.active().span span.log_kv({'event': 'sampleFileOpened', 'sampleTag': tag}) response = None try: - with opentracing.tracer.start_active_span('downloadSample', child_of=span): + with opentracing.tracer.start_active_span('magicDoer.downloadSample'): # Download Sample object_path = urljoin(Config.STORAGE_SERVICE_URL, f"object/{tag}") @@ -38,11 +39,11 @@ class MagicDoer: logging.debug(f"Downloaded sample to {sample_file_path}") - with opentracing.tracer.start_active_span('loadClassifier', child_of=span): + with opentracing.tracer.start_active_span('magicDoer.getClassifier'): # Get a classifier that uses the default model model_details, classifier = cls.classifier_cache.get_default_classifier() - with opentracing.tracer.start_active_span('runClassifier', child_of=span) as child_span: + with opentracing.tracer.start_active_span('magicDoer.runClassifier'): # do the majic classification_start_time = time.time() predicted_class_name, labeled_predictions = classifier.predict(sample_file_path) @@ -63,6 +64,7 @@ class MagicDoer: except FileNotFoundError: pass + span = opentracing.tracer.scope_manager.active().span span.log_kv({'event': 'sampleFileDeleted', 'sampleTag': tag}) if not response: diff --git a/cnn_classification_service/main.py b/cnn_classification_service/main.py index c28b37e..c6b2b99 100644 --- a/cnn_classification_service/main.py +++ b/cnn_classification_service/main.py @@ -19,16 +19,16 @@ from magic_doer import MagicDoer def message_callback(channel, method, properties, body): - with opentracing.tracer.start_active_span('messageHandling', finish_on_close=True) as span: + with opentracing.tracer.start_active_span('main.handleMessage', finish_on_close=True) as scope: try: msg = json.loads(body.decode('utf-8')) except (UnicodeDecodeError, json.JSONDecodeError) as e: logging.warning(f"Invalid message recieved: {e}") return - span.log_kv({'event': 'messageParsed', 'sampleTag': msg['tag']}) + scope.span.log_kv({'event': 'messageParsed', 'sampleTag': msg['tag']}) - with opentracing.tracer.start_active_span('runAlgorithm', child_of=span) as child_span: + with opentracing.tracer.start_active_span('magicDoer.run_everything'): results = MagicDoer.run_everything(msg) # <- This is where the magic happens if results: