From f7c0df98b21971180313c05c57607bc27c17c467 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gi=C3=B2=20Diani?= Date: Thu, 16 Jan 2025 22:18:12 +0100 Subject: [PATCH] improvement topic modelling --- VANA-php/app/Srgssr/Episode.php | 2 +- VANA-php/app/Srgssr/Subtitles.php | 17 +- .../2025_01_16_210341_add_topics.php | 26 + VANA-php/resources/css/app.css | 26 + VANA-php/resources/views/detail.blade.php | 3 + VANA-python/database/src/database/queries.py | 5 + .../.gitattributes | 0 .../.gitignore | 0 VANA-python/subtitles_processing/README.md | 21 + .../pixi.lock | 1078 +- .../pyproject.toml | 6 +- .../src/subtitles_processing}/__init__.py | 0 .../__pycache__/normalize_vtt.cpython-311.pyc | Bin 4019 -> 4007 bytes .../__pycache__/timecode.cpython-311.pyc | Bin .../src/subtitles_processing}/count_words.py | 0 .../subtitles_processing}/normalize_vtt.py | 5 +- .../sentence_sentiment.py | 2 +- .../subtitles-processing.py | 0 .../src/subtitles_processing}/timecode.py | 0 .../src/subtitles_processing/topics.py | 63 + VANA-python/video_deepface/.gitattributes | 2 - VANA-python/video_deepface/.gitignore | 4 - VANA-python/video_deepface/pixi.lock | 4953 ----- VANA-python/video_deepface/pyproject.toml | 23 - VANA-python/video_deepface/src/analyze.py | 10 - .../src/video_deepface/__init__.py | 0 VANA-python/video_deepface2/.gitattributes | 2 - VANA-python/video_deepface2/.gitignore | 4 - VANA-python/video_deepface2/pixi.lock | 1276 -- VANA-python/video_deepface2/pyproject.toml | 20 - VANA-python/video_deepface2/src/test.py | 9 - .../src/video_deepface2/__init__.py | 0 VANA.sublime-project | 2 +- VANA.sublime-workspace | 18293 ++++++++-------- database.sqlite | Bin 5885952 -> 7938048 bytes 35 files changed, 10498 insertions(+), 15354 deletions(-) create mode 100644 VANA-php/database/migrations/2025_01_16_210341_add_topics.php rename VANA-python/{normalize_subtitles => subtitles_processing}/.gitattributes (100%) rename VANA-python/{normalize_subtitles => subtitles_processing}/.gitignore (100%) create mode 100644 VANA-python/subtitles_processing/README.md rename VANA-python/{normalize_subtitles => subtitles_processing}/pixi.lock (78%) rename VANA-python/{normalize_subtitles => subtitles_processing}/pyproject.toml (80%) rename VANA-python/{normalize_subtitles/src/normalize_subtitles => subtitles_processing/src/subtitles_processing}/__init__.py (100%) rename VANA-python/{normalize_subtitles/src/normalize_subtitles => subtitles_processing/src/subtitles_processing}/__pycache__/normalize_vtt.cpython-311.pyc (69%) rename VANA-python/{normalize_subtitles/src/normalize_subtitles => subtitles_processing/src/subtitles_processing}/__pycache__/timecode.cpython-311.pyc (100%) rename VANA-python/{normalize_subtitles/src/normalize_subtitles => subtitles_processing/src/subtitles_processing}/count_words.py (100%) rename VANA-python/{normalize_subtitles/src/normalize_subtitles => subtitles_processing/src/subtitles_processing}/normalize_vtt.py (95%) rename VANA-python/{normalize_subtitles/src/normalize_subtitles => subtitles_processing/src/subtitles_processing}/sentence_sentiment.py (92%) rename VANA-python/{normalize_subtitles/src/normalize_subtitles => subtitles_processing/src/subtitles_processing}/subtitles-processing.py (100%) rename VANA-python/{normalize_subtitles/src/normalize_subtitles => subtitles_processing/src/subtitles_processing}/timecode.py (100%) create mode 100644 VANA-python/subtitles_processing/src/subtitles_processing/topics.py delete mode 100644 VANA-python/video_deepface/.gitattributes delete mode 100644 VANA-python/video_deepface/.gitignore delete mode 100644 VANA-python/video_deepface/pixi.lock delete mode 100644 VANA-python/video_deepface/pyproject.toml delete mode 100644 VANA-python/video_deepface/src/analyze.py delete mode 100644 VANA-python/video_deepface/src/video_deepface/__init__.py delete mode 100644 VANA-python/video_deepface2/.gitattributes delete mode 100644 VANA-python/video_deepface2/.gitignore delete mode 100644 VANA-python/video_deepface2/pixi.lock delete mode 100644 VANA-python/video_deepface2/pyproject.toml delete mode 100644 VANA-python/video_deepface2/src/test.py delete mode 100644 VANA-python/video_deepface2/src/video_deepface2/__init__.py diff --git a/VANA-php/app/Srgssr/Episode.php b/VANA-php/app/Srgssr/Episode.php index 7eb4f13..9152400 100644 --- a/VANA-php/app/Srgssr/Episode.php +++ b/VANA-php/app/Srgssr/Episode.php @@ -47,7 +47,7 @@ class Episode extends Api $subtitles = new Subtitles; $subs = $subtitles->getWebVTT('urn:srf:episode:tv:'.$ep['id']); - $test = EpisodeModel::firstOrCreate( + $epModel = EpisodeModel::updateOrCreate( [ 'urn' => $ep['fullLengthUrn'] ], diff --git a/VANA-php/app/Srgssr/Subtitles.php b/VANA-php/app/Srgssr/Subtitles.php index a8a1289..23d4bda 100644 --- a/VANA-php/app/Srgssr/Subtitles.php +++ b/VANA-php/app/Srgssr/Subtitles.php @@ -18,13 +18,13 @@ class Subtitles extends Api public function getUrl(string $urn): string { + $response = Http::withHeaders($this->headers)->withQueryParameters([ 'episode' => $urn, ])->get($this->endpoint.'subtitles'); - - if($response->ok()){ - return $response->json()[0]['url']; + if($response->ok() && !empty($response[0]['url'])){ + return $response[0]['url']; } return false; @@ -34,9 +34,16 @@ class Subtitles extends Api public function getWebVTT(string $urn): ?string { $url = $this->getUrl($urn); - $response = Http::get($url); + $response = null; - if($response->successful()){ + if($url){ + $response = Http::get($url); + } else { + $altUrn = explode(":", $urn)[4]; + $response = Http::get("https://subtitles.eai-general.aws.srf.ch/srf/{$altUrn}/episode/de/vod/vod.vtt"); + } + + if($response && $response->successful()){ return $response->body(); } diff --git a/VANA-php/database/migrations/2025_01_16_210341_add_topics.php b/VANA-php/database/migrations/2025_01_16_210341_add_topics.php new file mode 100644 index 0000000..421f2cf --- /dev/null +++ b/VANA-php/database/migrations/2025_01_16_210341_add_topics.php @@ -0,0 +1,26 @@ +json('topics')->nullable(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + // + } +}; diff --git a/VANA-php/resources/css/app.css b/VANA-php/resources/css/app.css index 5354718..321b444 100644 --- a/VANA-php/resources/css/app.css +++ b/VANA-php/resources/css/app.css @@ -47,6 +47,17 @@ input, button, textarea, select { font: inherit; } +input{ + margin-top: .2em; + border-radius: .2em; + border: 1px solid #fff; + background: var(--c3); + color: #fff; + padding: .2em .5em; + display: block; +} + + h1, h2, h3, @@ -294,6 +305,21 @@ main { background: var(--c3); } +.track-ctrl h2{ + font-size: 1.2em; +} + +.track-ctrl p { + margin-top: 1em; + font-size: 1em; + line-height: 1.5; +} + +.track-ctrl label { + margin-top: 1em; + display: inline-block; +} + .track-viz{ background: var(--c3); } diff --git a/VANA-php/resources/views/detail.blade.php b/VANA-php/resources/views/detail.blade.php index 48daa6a..5ed17f4 100644 --- a/VANA-php/resources/views/detail.blade.php +++ b/VANA-php/resources/views/detail.blade.php @@ -116,7 +116,10 @@

Topics

+