diff --git a/.gitea/workflows/dockerfile_testing_runner.yaml b/.gitea/workflows/dockerfile_testing_runner.yaml index 1d5b24a..d7e7519 100644 --- a/.gitea/workflows/dockerfile_testing_runner.yaml +++ b/.gitea/workflows/dockerfile_testing_runner.yaml @@ -52,11 +52,10 @@ jobs: build-args: | ALLOWED_HOSTS='django-testing.tryton.geeraert.eu' DEBUG='True' + CACHE_BUST=${{ env.next_version }} tags: | gitea.tryton.geeraert.eu/pi-ux-ce/django-testing:latest gitea.tryton.geeraert.eu/pi-ux-ce/django-testing:${{ env.next_version }} - cache-from: type=registry,ref=gitea.tryton.geeraert.eu/pi-ux-ce/django-testing:buildcache - cache-to: type=registry,ref=gitea.tryton.geeraert.eu/pi-ux-ce/django-testing:buildcache,mode=max - name: update version in the variable run: | diff --git a/README.md b/README.md index c2e17a3..ce385ad 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ python django_code/manage.py makemigrations simpsons python django_code/manage.py makemigrations job python django_code/manage.py makemigrations telegram python django_code/manage.py makemigrations podcast +python django_code/manage.py makemigrations streaming python django_code/manage.py migrate python django_code/manage.py migrate --database=cloudmanager_db @@ -19,6 +20,7 @@ python django_code/manage.py migrate --database=redirect_db python django_code/manage.py migrate --database=job_db python django_code/manage.py migrate --database=telegram_db python django_code/manage.py migrate --database=podcast_db +python django_code/manage.py migrate --database=streaming_db ``` ### Database debug @@ -30,6 +32,7 @@ create database simpsons; create database job; create database telegram; create database podcast; +create database streaming; @@ -39,6 +42,7 @@ drop database simpsons; drop database job; drop database telegram; drop database podcast; +drop database streaming; ``` diff --git a/django_code/apps/streaming/__init__.py b/django_code/apps/streaming/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/django_code/apps/streaming/admin.py b/django_code/apps/streaming/admin.py new file mode 100644 index 0000000..ce2f06f --- /dev/null +++ b/django_code/apps/streaming/admin.py @@ -0,0 +1,35 @@ +from django.contrib import admin +from import_export.admin import ImportExportModelAdmin +from .models import streamingData +from .utils import full_testing + + + +@admin.action(description="check the URLs") +def check(modeladmin, request, queryset): + for query in queryset: + website_url = query.url + status=full_testing(website_url)[0] + result= full_testing(website_url)[1] + print(str(query.url)+": "+str(status)+" -> "+str(result)) + query.status = status + query.result = result + query.save(update_fields=["status","result"]) + + +# modeladmin.message_user( +# request, +# ngettext( +# "%d episode was successfully marked as seen.", +# "%d episodes were successfully marked as seen.", +# updated, +# ) +# % updated, +# messages.SUCCESS, +# ) + +@admin.register(streamingData) +class streamingAdmin(ImportExportModelAdmin): + list_display = ['url','result','status'] + fields = ['url'] + actions = [check] \ No newline at end of file diff --git a/django_code/apps/streaming/apps.py b/django_code/apps/streaming/apps.py new file mode 100644 index 0000000..85d22eb --- /dev/null +++ b/django_code/apps/streaming/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class StreamingConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'apps.streaming' diff --git a/django_code/apps/streaming/migrations/0001_initial.py b/django_code/apps/streaming/migrations/0001_initial.py new file mode 100644 index 0000000..a8da596 --- /dev/null +++ b/django_code/apps/streaming/migrations/0001_initial.py @@ -0,0 +1,26 @@ +# Generated by Django 4.2.30 on 2026-08-05 12:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='streamingData', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('url', models.CharField(max_length=1000)), + ('addedDate', models.DateField(auto_now_add=True, null=True, verbose_name='Published')), + ('status', models.CharField(blank=True, max_length=1000, null=True)), + ], + options={ + 'db_table': 'streaming', + }, + ), + ] diff --git a/django_code/apps/streaming/migrations/0002_streamingdata_result.py b/django_code/apps/streaming/migrations/0002_streamingdata_result.py new file mode 100644 index 0000000..ef814da --- /dev/null +++ b/django_code/apps/streaming/migrations/0002_streamingdata_result.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.30 on 2026-08-05 13:44 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('streaming', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='streamingdata', + name='result', + field=models.CharField(blank=True, max_length=1000, null=True), + ), + ] diff --git a/django_code/apps/streaming/migrations/__init__.py b/django_code/apps/streaming/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/django_code/apps/streaming/models.py b/django_code/apps/streaming/models.py new file mode 100644 index 0000000..640f865 --- /dev/null +++ b/django_code/apps/streaming/models.py @@ -0,0 +1,11 @@ +from django.db import models + +# Create your models here. +class streamingData(models.Model): + url = models.CharField(max_length=1000) + addedDate = models.DateField('Published', blank=True, null=True, auto_now_add=True) + status = models.CharField(max_length=1000, blank=True, null=True) + result = models.CharField(max_length=1000, blank=True, null=True) + + class Meta: + db_table = 'streaming' diff --git a/django_code/apps/streaming/tests.py b/django_code/apps/streaming/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/django_code/apps/streaming/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/django_code/apps/streaming/utils.py b/django_code/apps/streaming/utils.py new file mode 100644 index 0000000..b873d5f --- /dev/null +++ b/django_code/apps/streaming/utils.py @@ -0,0 +1,38 @@ +import requests +from django.conf import settings + + +def full_testing(url_website): + status = "unknown" + + #step 1 : test without VPN + status = test_website(url_website) + + if status == 200: + result = "REACHABLE" + else: + #step 2 : test with VPN + status = test_website(url_website,bool_proxy=True) + if status != 200: + result = "UNREACHABLE" + else: + result = "VPN REQUIRED" + + return status,result + + +def test_website(url_website,bool_proxy=False): + try: + if bool_proxy: + status = requests.get(url_website,proxies=settings.PROXIES).status_code + else: + status = requests.get(url_website).status_code + + except requests.exceptions.ConnectionError: + status = "ConnectionError" + + #identify the exception name + except Exception as err: + print(f"{type(err).__name__} was raised: {err}") + status = "exception not targeted" + return status \ No newline at end of file diff --git a/django_code/apps/streaming/views.py b/django_code/apps/streaming/views.py new file mode 100644 index 0000000..91ea44a --- /dev/null +++ b/django_code/apps/streaming/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/django_code/apps/website_redirection/admin.py b/django_code/apps/website_redirection/admin.py index b3172de..92aebb7 100644 --- a/django_code/apps/website_redirection/admin.py +++ b/django_code/apps/website_redirection/admin.py @@ -2,6 +2,8 @@ from django.contrib import admin from .models import historicalData,redirectData from .utils import redirect_method from import_export.admin import ImportExportModelAdmin +from django.utils.html import format_html + # Register your models here. @@ -18,14 +20,17 @@ def re_redirect(modeladmin, request, queryset): request, output_generated ) - @admin.register(historicalData) class historicalAdmin(ImportExportModelAdmin): - list_display = [field.name for field in historicalData._meta.fields if field.name != 'id'] + list_input = (field.name for field in historicalData._meta.fields if field.name != 'id' or field.name != 'url') + list_display = ["clickable_links"] + list_display.extend(list_input) actions = [re_redirect] - + + def clickable_links(self, obj): + return format_html('{}', obj.url,obj.url) + @admin.register(redirectData) class redirectAdmin(ImportExportModelAdmin): list_display = [field.name for field in redirectData._meta.fields if field.name != 'id'] - \ No newline at end of file diff --git a/django_code/apps/website_redirection/models.py b/django_code/apps/website_redirection/models.py index 3aa5857..e31c8bc 100644 --- a/django_code/apps/website_redirection/models.py +++ b/django_code/apps/website_redirection/models.py @@ -1,4 +1,6 @@ from django.db import models +from django.contrib import admin +from django.utils.html import format_html class historicalData(models.Model): url = models.CharField(max_length=1000) diff --git a/django_code/core/routers.py b/django_code/core/routers.py index 6199a21..cc62513 100644 --- a/django_code/core/routers.py +++ b/django_code/core/routers.py @@ -27,6 +27,9 @@ class ProjectDatabaseRouter: if model._meta.app_label == 'podcast': return 'podcast_db' + if model._meta.app_label == 'streaming': + return 'streaming_db' + return 'default' def db_for_write(self, model, **hints): @@ -50,14 +53,17 @@ class ProjectDatabaseRouter: if model._meta.app_label == 'podcast': return 'podcast_db' + if model._meta.app_label == 'streaming': + return 'streaming_db' + return 'default' def allow_relation(self, obj1, obj2, **hints): """Allow relations if both models are in the same database.""" # Prevents accidental foreign key relationships across different physical DBs if ( - obj1._meta.app_label in ['cloudmanager', 'simpsons', 'redirect', 'job', 'telegram', 'podcast'] or - obj2._meta.app_label in ['cloudmanager', 'simpsons', 'redirect', 'job', 'telegram', 'podcast'] + obj1._meta.app_label in ['cloudmanager', 'simpsons', 'redirect', 'job', 'telegram', 'podcast', 'streaming'] or + obj2._meta.app_label in ['cloudmanager', 'simpsons', 'redirect', 'job', 'telegram', 'podcast', 'streaming'] ): return obj1._meta.app_label == obj2._meta.app_label return None @@ -83,8 +89,11 @@ class ProjectDatabaseRouter: if app_label == 'podcast': return db == 'podcast_db' + if app_label == 'streaming': + return db == 'streaming_db' + # Ensure default Django apps (auth, sessions, etc.) don't migrate into simpsons/cloudmanager - if db in ['cloudmanager_db', 'simpsons_db', 'redirect_db', 'job_db', 'telegram_db', 'podcast_db']: + if db in ['cloudmanager_db', 'simpsons_db', 'redirect_db', 'job_db', 'telegram_db', 'podcast_db', 'streaming_db']: return False return None diff --git a/django_code/core/settings.py b/django_code/core/settings.py index e9ff76f..7ff2b56 100644 --- a/django_code/core/settings.py +++ b/django_code/core/settings.py @@ -11,6 +11,7 @@ https://docs.djangoproject.com/en/4.2/ref/settings/ """ import os from pathlib import Path +from urllib import request as urllib_request # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -49,6 +50,7 @@ INSTALLED_APPS = [ 'apps.job_dashboard.apps.JobDashboardConfig', 'apps.telegram_communication.apps.TelegramCommunicationConfig', 'apps.podcast_aggregator.apps.PodcastAggregatorConfig', + 'apps.streaming.apps.StreamingConfig', 'import_export', 'djangoql', ] @@ -153,6 +155,14 @@ DATABASES = { 'HOST': os.environ.get('DB_HOST'), 'PORT': os.environ.get('DB_PORT'), }, + 'streaming_db': { + 'ENGINE': 'django.db.backends.postgresql', + 'NAME': 'streaming', + 'USER': os.environ.get('DB_USERNAME'), + 'PASSWORD': os.environ.get('DB_PASSWORD'), + 'HOST': os.environ.get('DB_HOST'), + 'PORT': os.environ.get('DB_PORT'), + }, } DATABASE_ROUTERS = ['core.routers.ProjectDatabaseRouter'] #DATABASE_ROUTERS = [] @@ -214,3 +224,23 @@ CELERY_IMPORTS = [ 'apps.random_simpsons.utils' # Add other app tasks here if needed ] + + + +########################################################################################### +# +# Custom settings +# +########################################################################################### + + +PROXY_URL = 'http://gluetun-svc.gluetun.svc:8888' +PROXY = urllib_request.ProxyHandler({ + 'http': PROXY_URL, + 'https': PROXY_URL # Add the handler for HTTPS +}) + +PROXIES = { + "http" : PROXY_URL, + "https" : PROXY_URL +} diff --git a/django_code/core/urls.py b/django_code/core/urls.py index c8bb224..25d64ce 100644 --- a/django_code/core/urls.py +++ b/django_code/core/urls.py @@ -93,5 +93,4 @@ urlpatterns = [ ################### cloudmanager ################### path('podcast/', podcast_views.execute_podcast), - ] diff --git a/infra/production/entrypoint.sh b/infra/production/entrypoint.sh index 40ab7be..f501c9e 100644 --- a/infra/production/entrypoint.sh +++ b/infra/production/entrypoint.sh @@ -7,6 +7,7 @@ python django_code/manage.py makemigrations simpsons python django_code/manage.py makemigrations job python django_code/manage.py makemigrations telegram python django_code/manage.py makemigrations podcast +python django_code/manage.py makemigrations streaming python django_code/manage.py migrate python django_code/manage.py migrate --database=cloudmanager_db @@ -15,6 +16,8 @@ python django_code/manage.py migrate --database=redirect_db python django_code/manage.py migrate --database=job_db python django_code/manage.py migrate --database=telegram_db python django_code/manage.py migrate --database=podcast_db +python django_code/manage.py migrate --database=streaming_db + python django_code/manage.py runserver "0.0.0.0:8000" diff --git a/infra/testing/dockerfile b/infra/testing/dockerfile index ff5bd1c..6746664 100644 --- a/infra/testing/dockerfile +++ b/infra/testing/dockerfile @@ -15,19 +15,19 @@ RUN apt-get update && \ apt-get install -y git nano && \ rm -rf /var/lib/apt/lists/* +# CACHE_BUST forces this layer (and everything after it) to re-run on every +# build, otherwise Docker reuses a cached clone from a previous build and the +# image silently keeps stale code. +ARG CACHE_BUST=1 +RUN echo "cache bust: ${CACHE_BUST}" && \ + git clone --depth 1 -b testing https://gitea.tryton.geeraert.eu/pi-ux-ce/Django_kube.git . -# Copy only the requirements file first (Changes infrequently) -COPY requirements.txt ./ - -RUN pip install --cache-dir=/opt/hostedtoolcache/pip --upgrade pip +RUN pip install --cache-dir=/opt/hostedtoolcache/pip --upgrade pip RUN pip list --cache-dir=/opt/hostedtoolcache/pip --outdated RUN pip install --cache-dir=/opt/hostedtoolcache/pip --upgrade wheel RUN pip install --cache-dir=/opt/hostedtoolcache/pip --upgrade setuptools RUN pip install --cache-dir=/opt/hostedtoolcache/pip -r requirements.txt -# Copy the rest of your application code (Changes frequently, invalidates subsequent cache) -COPY . ./ - COPY infra/testing/entrypoint.sh /code/entrypoint.sh RUN chmod +x /code/entrypoint.sh diff --git a/infra/testing/entrypoint.sh b/infra/testing/entrypoint.sh index c8440e4..44f6586 100644 --- a/infra/testing/entrypoint.sh +++ b/infra/testing/entrypoint.sh @@ -1,14 +1,12 @@ #!/bin/bash +set -e -cd Django_kube - -#another methid but needs to rm -rf before -#git clone --depth 1 -b main https://gitea.tryton.geeraert.eu/pi-ux-ce/Django_kube.git -git stash -git pull +git pull origin testing pip install -r requirements.txt + + python django_code/manage.py makemigrations python django_code/manage.py makemigrations redirect python django_code/manage.py makemigrations cloudmanager @@ -16,6 +14,7 @@ python django_code/manage.py makemigrations simpsons python django_code/manage.py makemigrations job python django_code/manage.py makemigrations telegram python django_code/manage.py makemigrations podcast +python django_code/manage.py makemigrations streaming python django_code/manage.py migrate python django_code/manage.py migrate --database=cloudmanager_db @@ -24,6 +23,7 @@ python django_code/manage.py migrate --database=redirect_db python django_code/manage.py migrate --database=job_db python django_code/manage.py migrate --database=telegram_db python django_code/manage.py migrate --database=podcast_db +python django_code/manage.py migrate --database=streaming_db python django_code/manage.py runserver "0.0.0.0:8000" diff --git a/requirements.txt b/requirements.txt index 137d398..6bd0bf5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,3 +19,4 @@ bitwarden_sdk xmltodict whitenoise djangoql +python-json-logger