addign this for the syntax but this solution is too slow

.gitea/python_build/dockerfile

@@ -0,0 +1,16 @@
+FROM gitea.tryton.geeraert.eu/pi-ux-ce/python:3
+
+ENV PYTHONUNBUFFERED 1
+
+WORKDIR /code/
+
+# Copy only the requirements file first (Changes infrequently)
+COPY requirements.txt ./
+
+
+RUN --mount=type=cache,target=/root/.cache/pip pip install --upgrade pip 
+RUN --mount=type=cache,target=/root/.cache/pip pip list --outdated
+RUN --mount=type=cache,target=/root/.cache/pip pip install --upgrade wheel
+RUN --mount=type=cache,target=/root/.cache/pip pip install --upgrade setuptools
+RUN --mount=type=cache,target=/root/.cache/pip pip install -r requirements.txt
+

.gitea/python_build/requirements.txt

@@ -0,0 +1,15 @@
+django
+psycopg2
+requests
+proxmoxer
+docker
+kubernetes
+ruamel.yaml
+pyyaml
+django-bootstrap-v5
+jinja2
+watchdog
+celery
+redis
+lxml
+requests

.gitea/workflows/dockerfile_runner.yaml

@@ -2,10 +2,89 @@ name: Gitea Actions Demo
 run-name: ${{ gitea.actor }} is testing out Gitea Actions 🚀
 on:
   push:
+  #paths-ignore:
+  #  - 'requirements.txt'
   issue_comment:
     types: [created]
 
 jobs:
+  # Job to check for file changes and set an output
+  changes:
+    runs-on: ubuntu-latest
+    outputs:
+      requirements_changed: ${{ steps.filter.outputs.requirements }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          filters: |
+            requirements:
+              - 'requirements.txt'
+
+  Build-dockerfile:
+    runs-on: ubuntu-latest
+    needs: changes
+    if: ${{ needs.changes.outputs.requirements_changed == 'true' }}
+    steps:
+      - run: echo "🔎 The name of your branch is ${{ gitea.ref }} and your repository is ${{ gitea.repository }}."
+      - name: Check out repository code
+        uses: actions/checkout@v4
+      - run: echo "💡 The ${{ gitea.repository }} repository has been cloned to the runner."
+      - name: List files in the repository
+        run: |
+          ls ${{ gitea.workspace }}
+
+      - name: Compute next version
+        id: versioning
+        run: |
+          latest="${{ vars.PYTHON_VERSION }}"
+          next_version=$(echo "$latest" | awk -F. -v OFS=. '{$NF += 1 ; print}')
+          echo "next_version=$next_version" >> $GITHUB_ENV
+          echo "Variables used :"
+          echo "------------------------------------------------"
+          echo "Next version: $next_version"
+
+      - uses: actions/cache@v3
+        with:
+          path: /root/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+
+      - name: inject cache into docker
+        uses: reproducible-containers/buildkit-cache-dance@v3.1.0
+        with:
+          cache-source: /root/.cache/pip
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to gitea
+        uses: docker/login-action@v3
+        with:
+          registry: gitea.tryton.geeraert.eu
+          username: ${{ secrets.USERNAME_GITEA }}
+          password: ${{ secrets.PASSWORD_GITEA }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: "{{defaultContext}}:.gitea/python_build"
+          push: true
+          tags: |
+            gitea.tryton.geeraert.eu/pi-ux-ce/modified-python:latest
+            gitea.tryton.geeraert.eu/pi-ux-ce/modified-python:${{ env.next_version }}
+          cache-from: type=registry,ref=gitea.tryton.geeraert.eu/pi-ux-ce/modified-python:buildcache
+          cache-to: type=registry,ref=gitea.tryton.geeraert.eu/pi-ux-ce/modified-python:buildcache,mode=max
+
+      - name: update version in the variable
+        run: |
+          curl  -X PUT -H "Content-Type: application/json" \
+          --user ${{ secrets.USERNAME_GITEA }}:${{ secrets.PASSWORD_GITEA }} \
+          https://gitea.tryton.geeraert.eu/api/v1/repos/pi-ux-ce/${{ github.event.repository.name }}/actions/variables/PYTHON_VERSION \
+          -d "{\"Value\":\"$next_version\"}"
   Explore-Gitea-Actions:
     runs-on: ubuntu-latest
     steps:
@@ -27,12 +106,13 @@ jobs:
           echo "------------------------------------------------"
           echo "Next version: $next_version"
 
-      - name: Cache pip
-        uses: actions/cache@v3
+      - uses: actions/cache@v3
         with:
           path: /root/.cache/pip
-          key: pip
-          restore-keys: pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
 
       - name: inject cache into docker
         uses: reproducible-containers/buildkit-cache-dance@v3.1.0

.gitignore

@@ -2,3 +2,5 @@
 */.idea
 venv/*
 *__pycache__*
+Webcal-python/*
+.idea/*

django_code/stash_management/apps.py

@@ -16,6 +16,11 @@ import mimetypes
 import ruamel.yaml
 import yaml
 from urllib import request as urlrequest
+from urllib import parse as urlparse
+
+
+
+
 
 proxy_url = 'http://gluetun-svc.gluetun.svc:8888'
 proxy = urlrequest.ProxyHandler({
@@ -23,7 +28,12 @@ proxy = urlrequest.ProxyHandler({
     'https': proxy_url  # Add the handler for HTTPS
 })
 
-destination_folder = "/tmp/stash" 
+proxies = { 
+              "http"  : proxy_url, 
+              "https" : proxy_url
+            }
+
+destination_folder = "/tmp/stash/" 
 
 def hello(request):
     return HttpResponse('<h1>Hello stash!</h1>')
@@ -32,23 +42,45 @@ def form(request):
 def album(request):
     return render(request, 'album_form.html',{})
 
+def check_and_create_destination_folder(name):
+    #check if the output directory exist and create if not
+    if not os.path.isdir(destination_folder):
+       os.makedirs(destination_folder)
+      
+    #check if the named directory exist and create if not
+    destination_named_folder = os.path.join(destination_folder,name)
+    if not os.path.isdir(destination_named_folder):
+           os.makedirs(destination_named_folder)
+        
+    return destination_named_folder
+    
 def album_form(request):
     name = request.GET.get('name')
     url = request.GET.get('url')
     external_galleries = request.GET.get('external_galleries')
 
-    album_downloader(name=name,url=url,external_galleries=external_galleries)
+    album_downloader.delay(name=name,url=url,external_galleries=external_galleries)
     
     text_generated = "successfully sent to the album downloader"
     return HttpResponse(text_generated, content_type="text/plain")
-
+    
+@shared_task()
 def album_downloader(name,url,external_galleries):
     print(name+" "+url+" "+external_galleries)
+    
+    opener = urlrequest.build_opener(proxy)
+    urlrequest.install_opener(opener)
+    
     req = urlrequest.Request(url)
     urlcurl = urlrequest.urlopen(req).read()
-    page = requests.get(url)
+   
+    page = requests.get(url,proxies=proxies)
     webpage = html.fromstring(page.content)
     
+    #create the prerequisites folder
+    destination_named_folder = check_and_create_destination_folder(name=name)
+    
+    
     #get all URLs from the code source (searching for href tag on the code)
     #https://stackoverflow.com/questions/21455349/xpath-query-get-attribute-href-from-a-tag
     
@@ -57,18 +89,36 @@ def album_downloader(name,url,external_galleries):
     #search for the href that contains the galleries
     webpage_all_album_url = webpage.xpath("//a[contains(@href,'pornpics.com/galleries/')]/@href")
     #print(webpage_all_album_url[3])
-    #print(webpage_all_album_url)
+    #print(webpage_all_album_url) 
 
 
     for album in webpage_all_album_url:
         print("Album "+album)
         print("")
-        page = requests.get(album)
+        page = requests.get(album,proxies=proxies)
         webpage = html.fromstring(page.content)
-        print(webpage.xpath("//a[contains(@href,'cdni.pornpics.com/1280/')]/@href"))
+        
+        images_list = webpage.xpath("//a[contains(@href,'cdni.pornpics.com/1280/')]/@href")
+        images_downloader.delay(name=name,images_url_list=images_list,destination_named_folder=destination_named_folder)
+        print(images_list)
     return 0
+    
+@shared_task()
+def images_downloader(name,images_url_list,destination_named_folder):
+    for image_url in images_url_list:
+        split = urlparse.urlsplit(image_url)
+        #filename = destination_folder + split.path.split("/")[-1]
+        filename =   os.path.join(destination_named_folder,split.path.split("/")[-1])
 
 
+        print("downloading: "+filename+" -- "+image_url)
+        
+        opener = urlrequest.build_opener(proxy)
+        
+        urlrequest.install_opener(opener)
+        urlrequest.urlretrieve(image_url, filename )
+        
+    return 0
 
 def incremental(request):
     return render(request, 'incremental_form.html',{})

django_code/stash_management/templates/listing.html

@@ -10915,7 +10915,6 @@
             <div class="col mb-5">
                 <div class="card h-100">
                     <!-- Sale badge-->
-                    <div class="badge bg-dark text-white position-absolute" style="top: 0.5rem; right: 0.5rem">Sale</div>
                     <!-- Product image-->
                     <img class="card-img-top" src="https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcTUHI8m7S7bMhMC2WldjOFV4e9pjeK0m9asEw&s" alt="..." />
                     <!-- Product details-->
@@ -10941,7 +10940,7 @@
                     <div class="card-body p-4">
                         <div class="text-center">
                             <!-- Product name-->
-                            <h5 class="fw-bolder">Popular Item</h5>
+                            <h5 class="fw-bolder">Celeb CX ????</h5>
                             <!-- Product reviews-->
                             <div class="d-flex justify-content-center small text-warning mb-2">
                                 <div class="bi-star-fill"></div>
@@ -10951,7 +10950,7 @@
                                 <div class="bi-star-fill"></div>
                             </div>
                             <!-- Product price-->
-                            $40.00
+                            https://celeb.cx
                         </div>
                     </div>
                     <!-- Product actions-->
@@ -10963,17 +10962,15 @@
             <div class="col mb-5">
                 <div class="card h-100">
                     <!-- Sale badge-->
-                    <div class="badge bg-dark text-white position-absolute" style="top: 0.5rem; right: 0.5rem">Sale</div>
                     <!-- Product image-->
-                    <img class="card-img-top" src="https://dummyimage.com/450x300/dee2e6/6c757d.jpg" alt="..." />
+                    <img class="card-img-top" src="https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcRlAP1QLW33zVG3dF-EmGalwc7L0uMUJEzRBA&s" alt="..." />
                     <!-- Product details-->
                     <div class="card-body p-4">
                         <div class="text-center">
                             <!-- Product name-->
-                            <h5 class="fw-bolder">Sale Item</h5>
+                            <h5 class="fw-bolder">Reddit</h5>
                             <!-- Product price-->
-                            <span class="text-muted text-decoration-line-through">$50.00</span>
-                            $25.00
+                            https://www.reddit.com/
                         </div>
                     </div>
                     <!-- Product actions-->
@@ -10990,7 +10987,7 @@
                     <div class="card-body p-4">
                         <div class="text-center">
                             <!-- Product name-->
-                            <h5 class="fw-bolder">Fancy Product</h5>
+                            <h5 class="fw-bolder">Regifs</h5>
                             <!-- Product price-->
                             $120.00 - $280.00
                         </div>

docker-compose.yml

@@ -0,0 +1,11 @@
+services:
+  db:
+    image: django:local
+    restart: always
+    volumes:
+      - type: bind
+        target: /code
+        source: ./django_code
+    command: python manage.py runserver 0.0.0.0:7000
+    ports:
+      - 7000:7000

dockerfile

@@ -1,19 +1,9 @@
-FROM gitea.tryton.geeraert.eu/pi-ux-ce/python:3
+FROM gitea.tryton.geeraert.eu/pi-ux-ce/modified-python:latest
 
 ENV PYTHONUNBUFFERED 1
 
 WORKDIR /code/
 
-# Copy only the requirements file first (Changes infrequently)
-COPY requirements.txt ./
-
-
-RUN --mount=type=cache,target=/root/.cache/pip pip install --upgrade pip 
-RUN --mount=type=cache,target=/root/.cache/pip pip list --outdated
-RUN --mount=type=cache,target=/root/.cache/pip pip install --upgrade wheel
-RUN --mount=type=cache,target=/root/.cache/pip pip install --upgrade setuptools
-RUN --mount=type=cache,target=/root/.cache/pip pip install -r requirements.txt
-
 # Copy the rest of your application code (Changes frequently, invalidates subsequent cache)
 COPY . ./
 RUN python django_code/manage.py migrate

requirements.txt

@@ -12,4 +12,4 @@ watchdog
 celery
 redis
 lxml
-requests
+requests