addign this for the syntax but this solution is too slow

.gitea/python_build/dockerfile

@@ -0,0 +1,16 @@
+FROM gitea.tryton.geeraert.eu/pi-ux-ce/python:3
+
+ENV PYTHONUNBUFFERED 1
+
+WORKDIR /code/
+
+# Copy only the requirements file first (Changes infrequently)
+COPY requirements.txt ./
+
+
+RUN --mount=type=cache,target=/root/.cache/pip pip install --upgrade pip 
+RUN --mount=type=cache,target=/root/.cache/pip pip list --outdated
+RUN --mount=type=cache,target=/root/.cache/pip pip install --upgrade wheel
+RUN --mount=type=cache,target=/root/.cache/pip pip install --upgrade setuptools
+RUN --mount=type=cache,target=/root/.cache/pip pip install -r requirements.txt
+

.gitea/python_build/requirements.txt

@@ -0,0 +1,15 @@
+django
+psycopg2
+requests
+proxmoxer
+docker
+kubernetes
+ruamel.yaml
+pyyaml
+django-bootstrap-v5
+jinja2
+watchdog
+celery
+redis
+lxml
+requests

.gitea/workflows/dockerfile_runner.yaml

@@ -2,10 +2,89 @@ name: Gitea Actions Demo
 run-name: ${{ gitea.actor }} is testing out Gitea Actions 🚀
 on:
   push:
+  #paths-ignore:
+  #  - 'requirements.txt'
   issue_comment:
     types: [created]
 
 jobs:
+  # Job to check for file changes and set an output
+  changes:
+    runs-on: ubuntu-latest
+    outputs:
+      requirements_changed: ${{ steps.filter.outputs.requirements }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          filters: |
+            requirements:
+              - 'requirements.txt'
+
+  Build-dockerfile:
+    runs-on: ubuntu-latest
+    needs: changes
+    if: ${{ needs.changes.outputs.requirements_changed == 'true' }}
+    steps:
+      - run: echo "🔎 The name of your branch is ${{ gitea.ref }} and your repository is ${{ gitea.repository }}."
+      - name: Check out repository code
+        uses: actions/checkout@v4
+      - run: echo "💡 The ${{ gitea.repository }} repository has been cloned to the runner."
+      - name: List files in the repository
+        run: |
+          ls ${{ gitea.workspace }}
+
+      - name: Compute next version
+        id: versioning
+        run: |
+          latest="${{ vars.PYTHON_VERSION }}"
+          next_version=$(echo "$latest" | awk -F. -v OFS=. '{$NF += 1 ; print}')
+          echo "next_version=$next_version" >> $GITHUB_ENV
+          echo "Variables used :"
+          echo "------------------------------------------------"
+          echo "Next version: $next_version"
+
+      - uses: actions/cache@v3
+        with:
+          path: /root/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+
+      - name: inject cache into docker
+        uses: reproducible-containers/buildkit-cache-dance@v3.1.0
+        with:
+          cache-source: /root/.cache/pip
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to gitea
+        uses: docker/login-action@v3
+        with:
+          registry: gitea.tryton.geeraert.eu
+          username: ${{ secrets.USERNAME_GITEA }}
+          password: ${{ secrets.PASSWORD_GITEA }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: "{{defaultContext}}:.gitea/python_build"
+          push: true
+          tags: |
+            gitea.tryton.geeraert.eu/pi-ux-ce/modified-python:latest
+            gitea.tryton.geeraert.eu/pi-ux-ce/modified-python:${{ env.next_version }}
+          cache-from: type=registry,ref=gitea.tryton.geeraert.eu/pi-ux-ce/modified-python:buildcache
+          cache-to: type=registry,ref=gitea.tryton.geeraert.eu/pi-ux-ce/modified-python:buildcache,mode=max
+
+      - name: update version in the variable
+        run: |
+          curl  -X PUT -H "Content-Type: application/json" \
+          --user ${{ secrets.USERNAME_GITEA }}:${{ secrets.PASSWORD_GITEA }} \
+          https://gitea.tryton.geeraert.eu/api/v1/repos/pi-ux-ce/${{ github.event.repository.name }}/actions/variables/PYTHON_VERSION \
+          -d "{\"Value\":\"$next_version\"}"
   Explore-Gitea-Actions:
     runs-on: ubuntu-latest
     steps:
@@ -27,12 +106,13 @@ jobs:
           echo "------------------------------------------------"
           echo "Next version: $next_version"
 
-      - name: Cache pip
+      - uses: actions/cache@v3
-        uses: actions/cache@v3
         with:
           path: /root/.cache/pip
-          key: pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
-          restore-keys: pip
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
 
       - name: inject cache into docker
         uses: reproducible-containers/buildkit-cache-dance@v3.1.0

.gitignore

@@ -2,3 +2,5 @@
 */.idea
 venv/*
 *__pycache__*
+Webcal-python/*
+.idea/*

django_code/django_code/settings.py

@@ -56,7 +56,7 @@ TEMPLATES = [
     {
         'BACKEND': 'django.template.backends.django.DjangoTemplates',
         #'DIRS': ["/code/django_code/templates","/code/django_code/dashboard/templates/"],
-        'DIRS': [BASE_DIR / "templates","django_code/dashboard/templates/"],
+        'DIRS': [BASE_DIR / "templates","django_code/dashboard/templates/","django_code/stash_management/templates"],
         'APP_DIRS': True,
         'OPTIONS': {
             'context_processors': [

django_code/django_code/urls.py

@@ -24,6 +24,7 @@ from dashboard import apps as dashboardapp
 from dashboard import views as dashboardview
 from kubernetes_connection import apps as kubernetes_connection
 from stash_management import apps as stashapp
+from stash_management import views as stashview
 
 urlpatterns = [
 #    path('admin/', admin.site.urls),
@@ -59,11 +60,14 @@ urlpatterns = [
     path('kubernetes_connection/scale_down/<str:namespace_name>/<str:deploy_name>', kubernetes_connection.scale_down),
 
 ##################       kubernetes_connection#####
+    path('stash/', stashview.home),
     path('stash/hello/', stashapp.hello),
     path('stash/form/', stashapp.form),
-    path('stash/fapello/', stashapp.fapello),
+    path('stash/incremental/', stashapp.incremental),
+    path('stash/album/', stashapp.album),
+    path('stash/album-form/', stashapp.album_form, name='submit_form'),
     path('stash/submit-form/', stashapp.handle_survey_form, name='submit_form'),
-    path('stash/fapello-form/', stashapp.fapello_form, name='submit_form'),
+    path('stash/incremental-form/', stashapp.incremental_form, name='submit_form'),
 
 
 #    path('bookstack/', docker_connection.apps.hello),

django_code/stash_management/apps.py

@@ -7,6 +7,8 @@ from django.core.files import File
 from kubernetes import client, config
 from celery import shared_task
 
+from lxml import html
+import requests
 import os
 import time
 import random
@@ -14,6 +16,11 @@ import mimetypes
 import ruamel.yaml
 import yaml
 from urllib import request as urlrequest
+from urllib import parse as urlparse
+
+
+
+
 
 proxy_url = 'http://gluetun-svc.gluetun.svc:8888'
 proxy = urlrequest.ProxyHandler({
@@ -21,18 +28,102 @@ proxy = urlrequest.ProxyHandler({
     'https': proxy_url  # Add the handler for HTTPS
 })
 
-destination_folder = "/tmp/stash" 
+proxies = { 
+              "http"  : proxy_url, 
+              "https" : proxy_url
+            }
+
+destination_folder = "/tmp/stash/" 
 
 def hello(request):
     return HttpResponse('<h1>Hello stash!</h1>')
 def form(request):
     return render(request, 'stash_form.html',{})
+def album(request):
+    return render(request, 'album_form.html',{})
 
-def fapello(request):
+def check_and_create_destination_folder(name):
-    return render(request, 'fapello_form.html',{})
+    #check if the output directory exist and create if not
+    if not os.path.isdir(destination_folder):
+       os.makedirs(destination_folder)
+      
+    #check if the named directory exist and create if not
+    destination_named_folder = os.path.join(destination_folder,name)
+    if not os.path.isdir(destination_named_folder):
+           os.makedirs(destination_named_folder)
+        
+    return destination_named_folder
+    
+def album_form(request):
+    name = request.GET.get('name')
+    url = request.GET.get('url')
+    external_galleries = request.GET.get('external_galleries')
 
-def fapello_form(request):
+    album_downloader.delay(name=name,url=url,external_galleries=external_galleries)
-    #stash_folder = "/tmp/stash"
+    
+    text_generated = "successfully sent to the album downloader"
+    return HttpResponse(text_generated, content_type="text/plain")
+    
+@shared_task()
+def album_downloader(name,url,external_galleries):
+    print(name+" "+url+" "+external_galleries)
+    
+    opener = urlrequest.build_opener(proxy)
+    urlrequest.install_opener(opener)
+    
+    req = urlrequest.Request(url)
+    urlcurl = urlrequest.urlopen(req).read()
+   
+    page = requests.get(url,proxies=proxies)
+    webpage = html.fromstring(page.content)
+    
+    #create the prerequisites folder
+    destination_named_folder = check_and_create_destination_folder(name=name)
+    
+    
+    #get all URLs from the code source (searching for href tag on the code)
+    #https://stackoverflow.com/questions/21455349/xpath-query-get-attribute-href-from-a-tag
+    
+    webpage_all_url = webpage.xpath('//a/@href')
+    
+    #search for the href that contains the galleries
+    webpage_all_album_url = webpage.xpath("//a[contains(@href,'pornpics.com/galleries/')]/@href")
+    #print(webpage_all_album_url[3])
+    #print(webpage_all_album_url) 
+
+
+    for album in webpage_all_album_url:
+        print("Album "+album)
+        print("")
+        page = requests.get(album,proxies=proxies)
+        webpage = html.fromstring(page.content)
+        
+        images_list = webpage.xpath("//a[contains(@href,'cdni.pornpics.com/1280/')]/@href")
+        images_downloader.delay(name=name,images_url_list=images_list,destination_named_folder=destination_named_folder)
+        print(images_list)
+    return 0
+    
+@shared_task()
+def images_downloader(name,images_url_list,destination_named_folder):
+    for image_url in images_url_list:
+        split = urlparse.urlsplit(image_url)
+        #filename = destination_folder + split.path.split("/")[-1]
+        filename =   os.path.join(destination_named_folder,split.path.split("/")[-1])
+
+
+        print("downloading: "+filename+" -- "+image_url)
+        
+        opener = urlrequest.build_opener(proxy)
+        
+        urlrequest.install_opener(opener)
+        urlrequest.urlretrieve(image_url, filename )
+        
+    return 0
+
+def incremental(request):
+    return render(request, 'incremental_form.html',{})
+
+def incremental_form(request):
     name = request.GET.get('name')
     url = request.GET.get('url')
     url_suffix = request.GET.get('url_suffix')
@@ -62,8 +153,10 @@ def increment_and_download(minimum_increment,maximum_increment,url,url_suffix,na
     if not os.path.isdir(destination_named_folder):
            os.makedirs(destination_named_folder)
         
-    
+    print("\n downloading for "+name+" "+url+"XXX"+url_suffix)
+
     for increment in range(minimum_increment,maximum_increment):
+        print("increment : "+str(increment)+"/"+str(maximum_increment))
         if len(str(increment)) == 3:
             increment_generated ='0' + str(increment)
         elif len(str(increment)) == 2:
@@ -80,6 +173,7 @@ def increment_and_download(minimum_increment,maximum_increment,url,url_suffix,na
         urlrequest.urlretrieve(url_generated, destination_file )
         
         text_generated = "\n downloading for "+name+" "+url+increment_generated+url_suffix
+    print("download completed for "+name+" of "+str(maximum_increment)+" images")
     return text_generated
 def handle_survey_form(request):
     config.load_kube_config(config_file='/kubeconfig/config')

django_code/stash_management/templates/album_form.html

@@ -0,0 +1,288 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+    <title>Stash form</title>
+    <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto|Varela+Round">
+    <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.5.0/css/bootstrap.min.css">
+    <link rel="stylesheet" href="https://fonts.googleapis.com/icon?family=Material+Icons">
+    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/font-awesome/4.7.0/css/font-awesome.min.css">
+
+    <style>
+        @charset "UTF-8";
+        .m-b-xs {
+            margin-bottom: 2%;
+        }
+
+        .m-b-md {
+            margin-bottom: 10%;
+        }
+
+        .m-t-xs {
+            margin-top: 2%;
+        }
+
+        .m-t-sm {
+            margin-top: 5%;
+        }
+
+        * {
+            outline: 0 !important;
+        }
+
+        body {
+            padding: 2% 0;
+            background-color: #37404a;
+        }
+
+        h1 {
+            font-family: "Roboto", sans-serif;
+            font-size: 2.2em;
+            font-weight: 300;
+            color: #85ffc7;
+            text-transform: uppercase;
+        }
+
+        p {
+            font-family: "Roboto", sans-serif;
+            font-size: 1.1em;
+            font-weight: 300;
+            color: #fff;
+        }
+
+        a {
+            color: #fff;
+        }
+        a:hover {
+            text-decoration: none;
+            color: #fff;
+        }
+
+        #survey-form label,
+        #survey-form .labels {
+            display: block;
+            margin-bottom: 2%;
+            font-family: "Roboto", sans-serif;
+            font-size: 1.1em;
+            font-weight: 300;
+            color: #fff;
+            letter-spacing: 0.5px;
+        }
+        #survey-form input::-webkit-input-placeholder {
+            color: transparent !important;
+        }
+        #survey-form input::-moz-placeholder {
+            color: transparent !important;
+        }
+        #survey-form input:-ms-input-placeholder {
+            color: transparent !important;
+        }
+        #survey-form input:-moz-placeholder {
+            color: transparent !important;
+        }
+        #survey-form input,
+        #survey-form select {
+            display: block;
+            width: 100%;
+            overflow: hidden;
+            outline: none;
+            border: 2px solid #333;
+        }
+        #survey-form input {
+            margin-top: 1.5%;
+            padding: 0 0 5px 0;
+            background: transparent;
+            border: none;
+            outline: none;
+            border-bottom: 2px solid #fff;
+            font-size: 1.1em;
+            font-weight: 300;
+            color: #85ffc7;
+        }
+        #survey-form input:focus {
+            border-color: #85ffc7;
+        }
+        #survey-form [type=checkbox],
+        #survey-form [type=radio] {
+            display: inline-block;
+            width: auto;
+            margin: 0px 10px 0 0;
+        }
+        #survey-form [type=checkbox],
+        #survey-form [type=radio] {
+            display: inline-block;
+            margin: 0 10px 0 0 !important;
+            position: relative;
+            top: 5px;
+            right: 0;
+            bottom: 0;
+            left: 0;
+            -webkit-appearance: none;
+            -moz-appearance: none;
+            -o-appearance: none;
+            appearance: none;
+            height: 23px;
+            width: 23px;
+            transition: all 0.15s ease-out 0s;
+            background: #37404a;
+            color: #fff;
+            cursor: pointer;
+            outline: none;
+            z-index: 1000;
+            border: 1px solid #fff;
+        }
+        #survey-form [type=checkbox]:hover,
+        #survey-form [type=radio]:hover {
+            border-color: #85ffc7;
+        }
+        #survey-form [type=checkbox]:checked:before,
+        #survey-form [type=radio]:checked:before {
+            display: inline-block;
+            height: 21px;
+            width: 21px;
+            position: relative;
+            left: 0;
+            bottom: 0;
+            content: "";
+            text-align: center;
+            font-family: "Glyphicons Halflings";
+            line-height: 20px;
+            font-size: 15px;
+            color: #85ffc7;
+        }
+        #survey-form [type=checkbox]:focus,
+        #survey-form [type=radio]:focus {
+            outline: none;
+            border-color: #fff;
+        }
+        #survey-form [type=radio] {
+            border-radius: 50%;
+        }
+        #survey-form [type=radio]:after {
+            border-radius: 50%;
+        }
+        #survey-form [type=number] {
+            width: 45px;
+        }
+        #survey-form [type=number]::-webkit-inner-spin-button, #survey-form [type=number]::-webkit-outer-spin-button {
+            -webkit-appearance: none;
+            -moz-appearance: none;
+            appearance: none;
+            margin: 0;
+        }
+        #survey-form select {
+            height: 40px;
+            padding-left: 5px;
+            background-color: #37404a;
+            border: 2px solid #fff;
+            color: #85ffc7;
+        }
+        #survey-form select option {
+            padding: 5px 10px;
+            font-weight: 300;
+            cursor: pointer;
+        }
+        #survey-form select option:hover {
+            background-color: #85ffc7;
+        }
+        #survey-form textarea {
+            resize: none;
+            margin-top: 2%;
+            padding: 10px 10px 0px 20px;
+            width: 100%;
+            height: 90px;
+            color: #85ffc7;
+            background-color: #37404a;
+            border: 2px solid #fff;
+        }
+        #survey-form .btn {
+            display: inline-block;
+            position: relative;
+            width: 100%;
+            margin: 3% 0 0 0;
+            height: 45px;
+            text-transform: uppercase;
+            text-decoration: none;
+            cursor: pointer;
+            border: 3px solid #85ffc7;
+            border-radius: 0;
+            font-weight: 500;
+            font-size: 1.2em;
+            color: #85ffc7;
+            text-align: center;
+            background: none;
+            transition: color 0.25s ease;
+        }
+        #survey-form .btn:after {
+            position: absolute;
+            content: "";
+            top: 0;
+            left: 0;
+            width: 0;
+            height: 100%;
+            background-color: #85ffc7;
+            transform-origin: left;
+            transition: width 0.5s ease;
+            z-index: -1;
+        }
+        #survey-form .btn:hover {
+            color: #37404a;
+        }
+        #survey-form .btn:hover:after {
+            width: 100%;
+        }
+        #survey-form fieldset {
+            margin: 5% 0 0 0;
+        }
+
+        .copyright {
+            text-align: center;
+            font-size: 10px;
+            color: #fff;
+        }
+        .copyright i {
+            color: #ef5350;
+        }
+
+        @media only screen and (max-width: 600px) {
+            body {
+                padding: 8% 0;
+            }
+        }
+    </style>
+</head>
+<main id="main" class="container">
+    <div class="row">
+        <div class="col-xs-12 col-lg-offset-3 col-lg-6">
+            <div class="m-b-md text-center">
+                <h1 id="title">Stash Form</h1>
+                <p id="description" class="description" class="text-center">Download form</p>
+            </div>
+                <form method="GET" action="/stash/album-form/" id="survey-form" name="survey-form">
+
+                <fieldset>
+                    <label for="name" id="name-label">
+                        Name *
+                        <input class="" type="text" id="name" name="name" placeholder="Enter your name (required)" required />
+                    </label>
+                </fieldset>
+                <fieldset>
+                    <label for="url" id="url-label">
+                        URL *
+                        <input class="" type="url" id="url" name="url" placeholder="Enter the URL (required)" required />
+                    </label>
+                </fieldset>
+
+                <fieldset>
+                    <label for="external_galleries">
+                        Any external galleries ?
+                        <textarea id="external_galleries" name="external_galleries" ></textarea>
+                    </label>
+                </fieldset>
+                <button id="submit" type="submit" class="btn">Submit the form</button>
+            </form>
+        </div>
+    </div>
+</main>
+</html>

django_code/stash_management/templates/incremental_form.html

@@ -256,10 +256,10 @@
     <div class="row">
         <div class="col-xs-12 col-lg-offset-3 col-lg-6">
             <div class="m-b-md text-center">
-                <h1 id="title">fapello Form</h1>
+                <h1 id="title">Incremental Form</h1>
                 <p id="description" class="description" class="text-center">Download form</p>
             </div>
-                <form method="GET" action="/stash/fapello-form/" id="survey-form" name="survey-form">
+                <form method="GET" action="/stash/incremental-form/" id="survey-form" name="survey-form">
 
                 <fieldset>
                     <label for="name" id="name-label">

django_code/stash_management/views.py

@@ -5,4 +5,8 @@ from django.http import HttpResponse
 from django.shortcuts import render
 
 def hello(request):
-    return HttpResponse('<h1>Hello Django!</h1>')
+    return HttpResponse('<h1>Hello stash!</h1>')
+
+
+def home(request):
+    return render(request, 'listing.html')

docker-compose.yml

@@ -0,0 +1,11 @@
+services:
+  db:
+    image: django:local
+    restart: always
+    volumes:
+      - type: bind
+        target: /code
+        source: ./django_code
+    command: python manage.py runserver 0.0.0.0:7000
+    ports:
+      - 7000:7000

dockerfile

@@ -1,19 +1,9 @@
-FROM gitea.tryton.geeraert.eu/pi-ux-ce/python:3
+FROM gitea.tryton.geeraert.eu/pi-ux-ce/modified-python:latest
 
 ENV PYTHONUNBUFFERED 1
 
 WORKDIR /code/
 
-# Copy only the requirements file first (Changes infrequently)
-COPY requirements.txt ./
-
-
-RUN --mount=type=cache,target=/root/.cache/pip pip install --upgrade pip 
-RUN --mount=type=cache,target=/root/.cache/pip pip list --outdated
-RUN --mount=type=cache,target=/root/.cache/pip pip install --upgrade wheel
-RUN --mount=type=cache,target=/root/.cache/pip pip install --upgrade setuptools
-RUN --mount=type=cache,target=/root/.cache/pip pip install -r requirements.txt
-
 # Copy the rest of your application code (Changes frequently, invalidates subsequent cache)
 COPY . ./
 RUN python django_code/manage.py migrate

requirements.txt

@@ -11,4 +11,5 @@ jinja2
 watchdog
 celery
 redis
-os
+lxml
+requests