addign this for the syntax but this solution is too slow

Reviewed-on: #5

.gitea/python_build/dockerfile

@@ -0,0 +1,16 @@
+FROM gitea.tryton.geeraert.eu/pi-ux-ce/python:3
+
+ENV PYTHONUNBUFFERED 1
+
+WORKDIR /code/
+
+# Copy only the requirements file first (Changes infrequently)
+COPY requirements.txt ./
+
+
+RUN --mount=type=cache,target=/root/.cache/pip pip install --upgrade pip 
+RUN --mount=type=cache,target=/root/.cache/pip pip list --outdated
+RUN --mount=type=cache,target=/root/.cache/pip pip install --upgrade wheel
+RUN --mount=type=cache,target=/root/.cache/pip pip install --upgrade setuptools
+RUN --mount=type=cache,target=/root/.cache/pip pip install -r requirements.txt
+

.gitea/python_build/requirements.txt

@@ -0,0 +1,15 @@
+django
+psycopg2
+requests
+proxmoxer
+docker
+kubernetes
+ruamel.yaml
+pyyaml
+django-bootstrap-v5
+jinja2
+watchdog
+celery
+redis
+lxml
+requests

.gitea/workflows/dockerfile_runner.yaml

@@ -2,10 +2,89 @@ name: Gitea Actions Demo
 run-name: ${{ gitea.actor }} is testing out Gitea Actions 🚀
 on:
   push:
+  #paths-ignore:
+  #  - 'requirements.txt'
   issue_comment:
     types: [created]
 
 jobs:
+  # Job to check for file changes and set an output
+  changes:
+    runs-on: ubuntu-latest
+    outputs:
+      requirements_changed: ${{ steps.filter.outputs.requirements }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          filters: |
+            requirements:
+              - 'requirements.txt'
+
+  Build-dockerfile:
+    runs-on: ubuntu-latest
+    needs: changes
+    if: ${{ needs.changes.outputs.requirements_changed == 'true' }}
+    steps:
+      - run: echo "🔎 The name of your branch is ${{ gitea.ref }} and your repository is ${{ gitea.repository }}."
+      - name: Check out repository code
+        uses: actions/checkout@v4
+      - run: echo "💡 The ${{ gitea.repository }} repository has been cloned to the runner."
+      - name: List files in the repository
+        run: |
+          ls ${{ gitea.workspace }}
+
+      - name: Compute next version
+        id: versioning
+        run: |
+          latest="${{ vars.PYTHON_VERSION }}"
+          next_version=$(echo "$latest" | awk -F. -v OFS=. '{$NF += 1 ; print}')
+          echo "next_version=$next_version" >> $GITHUB_ENV
+          echo "Variables used :"
+          echo "------------------------------------------------"
+          echo "Next version: $next_version"
+
+      - uses: actions/cache@v3
+        with:
+          path: /root/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+
+      - name: inject cache into docker
+        uses: reproducible-containers/buildkit-cache-dance@v3.1.0
+        with:
+          cache-source: /root/.cache/pip
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to gitea
+        uses: docker/login-action@v3
+        with:
+          registry: gitea.tryton.geeraert.eu
+          username: ${{ secrets.USERNAME_GITEA }}
+          password: ${{ secrets.PASSWORD_GITEA }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: "{{defaultContext}}:.gitea/python_build"
+          push: true
+          tags: |
+            gitea.tryton.geeraert.eu/pi-ux-ce/modified-python:latest
+            gitea.tryton.geeraert.eu/pi-ux-ce/modified-python:${{ env.next_version }}
+          cache-from: type=registry,ref=gitea.tryton.geeraert.eu/pi-ux-ce/modified-python:buildcache
+          cache-to: type=registry,ref=gitea.tryton.geeraert.eu/pi-ux-ce/modified-python:buildcache,mode=max
+
+      - name: update version in the variable
+        run: |
+          curl  -X PUT -H "Content-Type: application/json" \
+          --user ${{ secrets.USERNAME_GITEA }}:${{ secrets.PASSWORD_GITEA }} \
+          https://gitea.tryton.geeraert.eu/api/v1/repos/pi-ux-ce/${{ github.event.repository.name }}/actions/variables/PYTHON_VERSION \
+          -d "{\"Value\":\"$next_version\"}"
   Explore-Gitea-Actions:
     runs-on: ubuntu-latest
     steps:
@@ -17,19 +96,29 @@ jobs:
         run: |
           ls ${{ gitea.workspace }}
 
-      - name: Cache pip
+      - name: Compute next version
-        uses: actions/cache@v3
+        id: versioning
+        run: |
+          latest="${{ vars.VERSION }}"
+          next_version=$(echo "$latest" | awk -F. -v OFS=. '{$NF += 1 ; print}')
+          echo "next_version=$next_version" >> $GITHUB_ENV
+          echo "Variables used :"
+          echo "------------------------------------------------"
+          echo "Next version: $next_version"
+
+      - uses: actions/cache@v3
         with:
           path: /root/.cache/pip
-          key: pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
-          restore-keys: pip
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
 
       - name: inject cache into docker
         uses: reproducible-containers/buildkit-cache-dance@v3.1.0
         with:
            cache-source: /root/.cache/pip
 
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
@@ -44,15 +133,30 @@ jobs:
         uses: docker/build-push-action@v6
         with:
           push: true
-        #build-contexts: pip_cache=${{steps.vars.outputs.pip-dir}}
+          tags: |
-          tags: gitea.tryton.geeraert.eu/pi-ux-ce/django-python:latest
+             gitea.tryton.geeraert.eu/pi-ux-ce/django-python:latest
+             gitea.tryton.geeraert.eu/pi-ux-ce/django-python:${{ env.next_version }}
           cache-from: type=registry,ref=gitea.tryton.geeraert.eu/pi-ux-ce/django-python:buildcache
           cache-to: type=registry,ref=gitea.tryton.geeraert.eu/pi-ux-ce/django-python:buildcache,mode=max
           
-      - name: drop the pod to force pulling the new image
+      - name: Rollout Django to force pulling the new image
         uses: actions-hub/kubectl@master
         env:
           KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
         with:
           args: rollout restart deployment django -n django
+          
+      - name: Rollout Celery worker to force pulling the new image
+        uses: actions-hub/kubectl@master
+        env:
+          KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
+        with:
+          args: rollout restart deployment celery-worker -n django
       - run: echo "🍏 This job's status is ${{ job.status }}."
+
+      - name: update version in the variable
+        run: |
+          curl  -X PUT -H "Content-Type: application/json" \
+          --user ${{ secrets.USERNAME_GITEA }}:${{ secrets.PASSWORD_GITEA }} \
+          https://gitea.tryton.geeraert.eu/api/v1/repos/pi-ux-ce/${{ github.event.repository.name }}/actions/variables/VERSION \
+          -d "{\"Value\":\"$next_version\"}"

.gitignore

@@ -2,3 +2,5 @@
 */.idea
 venv/*
 *__pycache__*
+Webcal-python/*
+.idea/*

django_code/django_code/__init__.py

@@ -0,0 +1,3 @@
+from .celery_settings import app as celery_app
+
+__all__ = ("celery_app",)

django_code/django_code/celery_settings.py

@@ -0,0 +1,8 @@
+import os
+from celery import Celery
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'django_code.settings')
+
+app = Celery("django_code")
+app.config_from_object("django.conf:settings", namespace="CELERY")
+app.autodiscover_tasks()

django_code/django_code/settings.py

@@ -56,7 +56,7 @@ TEMPLATES = [
     {
         'BACKEND': 'django.template.backends.django.DjangoTemplates',
         #'DIRS': ["/code/django_code/templates","/code/django_code/dashboard/templates/"],
-        'DIRS': [BASE_DIR / "templates","django_code/dashboard/templates/"],
+        'DIRS': [BASE_DIR / "templates","django_code/dashboard/templates/","django_code/stash_management/templates"],
         'APP_DIRS': True,
         'OPTIONS': {
             'context_processors': [
@@ -124,3 +124,11 @@ STATIC_URL = 'static/'
 
 DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
 #
+# Celery settings
+CELERY_BROKER_URL = "redis://django-redis-svc:6379"
+CELERY_RESULT_BACKEND = "redis://django-redis-svc:6379"
+
+CELERY_IMPORTS = [
+    'stash_management.apps', # This line imports the tasks.py file
+    # Add other app tasks here if needed
+]

django_code/django_code/urls.py

@@ -24,6 +24,7 @@ from dashboard import apps as dashboardapp
 from dashboard import views as dashboardview
 from kubernetes_connection import apps as kubernetes_connection
 from stash_management import apps as stashapp
+from stash_management import views as stashview
 
 urlpatterns = [
 #    path('admin/', admin.site.urls),
@@ -59,9 +60,14 @@ urlpatterns = [
     path('kubernetes_connection/scale_down/<str:namespace_name>/<str:deploy_name>', kubernetes_connection.scale_down),
 
 ##################       kubernetes_connection#####
+    path('stash/', stashview.home),
     path('stash/hello/', stashapp.hello),
     path('stash/form/', stashapp.form),
+    path('stash/incremental/', stashapp.incremental),
+    path('stash/album/', stashapp.album),
+    path('stash/album-form/', stashapp.album_form, name='submit_form'),
     path('stash/submit-form/', stashapp.handle_survey_form, name='submit_form'),
+    path('stash/incremental-form/', stashapp.incremental_form, name='submit_form'),
 
 
 #    path('bookstack/', docker_connection.apps.hello),

django_code/django_code/wsgi.py

@@ -14,3 +14,10 @@ from django.core.wsgi import get_wsgi_application
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'django_code.settings')
 
 application = get_wsgi_application()
+https_proxy = "gluetun-svc.gluetun.svc:8888"
+
+proxyDict = {
+    "https" : https_proxy,
+}
+
+#os.environ["PROXIES"] = proxyDict

django_code/stash_management/apps.py

@@ -2,45 +2,213 @@ from django.apps import AppConfig
 from django.http import HttpResponse
 from django.shortcuts import render
 from django.shortcuts import redirect
-from jinja2 import Environment, FileSystemLoader
+#from jinja2 import Environment, FileSystemLoader
 from django.core.files import File 
 from kubernetes import client, config
+from celery import shared_task
 
+from lxml import html
+import requests
+import os
 import time
 import random
 import mimetypes
 import ruamel.yaml
 import yaml
-import urllib.request
+from urllib import request as urlrequest
+from urllib import parse as urlparse
+
+
+
+
+
+proxy_url = 'http://gluetun-svc.gluetun.svc:8888'
+proxy = urlrequest.ProxyHandler({
+    'http': proxy_url,
+    'https': proxy_url  # Add the handler for HTTPS
+})
+
+proxies = { 
+              "http"  : proxy_url, 
+              "https" : proxy_url
+            }
+
+destination_folder = "/tmp/stash/" 
 
 def hello(request):
     return HttpResponse('<h1>Hello stash!</h1>')
 def form(request):
     return render(request, 'stash_form.html',{})
+def album(request):
+    return render(request, 'album_form.html',{})
 
+def check_and_create_destination_folder(name):
+    #check if the output directory exist and create if not
+    if not os.path.isdir(destination_folder):
+       os.makedirs(destination_folder)
+      
+    #check if the named directory exist and create if not
+    destination_named_folder = os.path.join(destination_folder,name)
+    if not os.path.isdir(destination_named_folder):
+           os.makedirs(destination_named_folder)
+        
+    return destination_named_folder
+    
+def album_form(request):
+    name = request.GET.get('name')
+    url = request.GET.get('url')
+    external_galleries = request.GET.get('external_galleries')
+
+    album_downloader.delay(name=name,url=url,external_galleries=external_galleries)
+    
+    text_generated = "successfully sent to the album downloader"
+    return HttpResponse(text_generated, content_type="text/plain")
+    
+@shared_task()
+def album_downloader(name,url,external_galleries):
+    print(name+" "+url+" "+external_galleries)
+    
+    opener = urlrequest.build_opener(proxy)
+    urlrequest.install_opener(opener)
+    
+    req = urlrequest.Request(url)
+    urlcurl = urlrequest.urlopen(req).read()
+   
+    page = requests.get(url,proxies=proxies)
+    webpage = html.fromstring(page.content)
+    
+    #create the prerequisites folder
+    destination_named_folder = check_and_create_destination_folder(name=name)
+    
+    
+    #get all URLs from the code source (searching for href tag on the code)
+    #https://stackoverflow.com/questions/21455349/xpath-query-get-attribute-href-from-a-tag
+    
+    webpage_all_url = webpage.xpath('//a/@href')
+    
+    #search for the href that contains the galleries
+    webpage_all_album_url = webpage.xpath("//a[contains(@href,'pornpics.com/galleries/')]/@href")
+    #print(webpage_all_album_url[3])
+    #print(webpage_all_album_url) 
+
+
+    for album in webpage_all_album_url:
+        print("Album "+album)
+        print("")
+        page = requests.get(album,proxies=proxies)
+        webpage = html.fromstring(page.content)
+        
+        images_list = webpage.xpath("//a[contains(@href,'cdni.pornpics.com/1280/')]/@href")
+        images_downloader.delay(name=name,images_url_list=images_list,destination_named_folder=destination_named_folder)
+        print(images_list)
+    return 0
+    
+@shared_task()
+def images_downloader(name,images_url_list,destination_named_folder):
+    for image_url in images_url_list:
+        split = urlparse.urlsplit(image_url)
+        #filename = destination_folder + split.path.split("/")[-1]
+        filename =   os.path.join(destination_named_folder,split.path.split("/")[-1])
+
+
+        print("downloading: "+filename+" -- "+image_url)
+        
+        opener = urlrequest.build_opener(proxy)
+        
+        urlrequest.install_opener(opener)
+        urlrequest.urlretrieve(image_url, filename )
+        
+    return 0
+
+def incremental(request):
+    return render(request, 'incremental_form.html',{})
+
+def incremental_form(request):
+    name = request.GET.get('name')
+    url = request.GET.get('url')
+    url_suffix = request.GET.get('url_suffix')
+    maximum_increment = int(request.GET.get('maximum_increment'))
+    minimum_increment = request.GET.get('minimum_increment')
+
+    increment_and_download.delay(minimum_increment=minimum_increment,maximum_increment=maximum_increment,url=url,url_suffix=url_suffix,name=name)
+
+    text_generated = "successfully sent to celery"
+    return HttpResponse(text_generated, content_type="text/plain")
+    
+@shared_task()
+def increment_and_download(minimum_increment,maximum_increment,url,url_suffix,name):
+    if minimum_increment == None:
+        minimum_increment = 1
+
+    text_generated = ""
+    minimum_increment = int(minimum_increment)
+    maximum_increment = int(maximum_increment)
+    
+    #check if the output directory exist and create if not
+    if not os.path.isdir(destination_folder):
+       os.makedirs(destination_folder)
+      
+    #check if the named directory exist and create if not
+    destination_named_folder = os.path.join(destination_folder,name)
+    if not os.path.isdir(destination_named_folder):
+           os.makedirs(destination_named_folder)
+        
+    print("\n downloading for "+name+" "+url+"XXX"+url_suffix)
+
+    for increment in range(minimum_increment,maximum_increment):
+        print("increment : "+str(increment)+"/"+str(maximum_increment))
+        if len(str(increment)) == 3:
+            increment_generated ='0' + str(increment)
+        elif len(str(increment)) == 2:
+            increment_generated ='00' + str(increment)
+        elif len(str(increment)) == 1:
+            increment_generated ='000' + str(increment)
+
+        url_generated = url+increment_generated+url_suffix
+        destination_filename = increment_generated+url_suffix
+        destination_file = os.path.join(destination_named_folder,destination_filename)
+        
+        opener = urlrequest.build_opener(proxy)
+        urlrequest.install_opener(opener)
+        urlrequest.urlretrieve(url_generated, destination_file )
+        
+        text_generated = "\n downloading for "+name+" "+url+increment_generated+url_suffix
+    print("download completed for "+name+" of "+str(maximum_increment)+" images")
+    return text_generated
 def handle_survey_form(request):
     config.load_kube_config(config_file='/kubeconfig/config')
     v1 = client.BatchV1Api()
     name = request.GET.get('name')
     url = request.GET.get('url')
-    external_galleries = request.GET.get('external_galleries')
+    external_galleries = str(request.GET.get('external_galleries'))
+    
+    stash_folder = "/stash"
+
+    if external_galleries:
+        print("external_galleries is not null")
+        performer_external_galleries = stash_folder+"/"+name+"_external_galleries.txt"
+        with open(performer_external_galleries, "a") as f:
+           f.write(external_galleries)
+        skip_searching_gallery = True
+    else:
+        print("external_galleries is null")
+        skip_searching_gallery = False
 
     with open('django_code/stash_management/templates/template.stash-pornpics-dowloader.job.yml', 'r') as file:
        #get the manifest in a variable template_manifest from the manifest file
        template_manifest = yaml.safe_load(file)
 
        #edit the container name 
-       template_manifest['metadata']['name'] ="stash-pornpics-dowloader"+name
+       template_manifest['metadata']['name'] ="stash-pornpics-dowloader-"+name
        template_manifest['spec']['template']['spec']['containers'][0]['name'] ="stash-pornpics-dowloader-"+name
 
        #edit the command based on the input
-       command=['bash', '/config/download.sh', '--http_proxy', 'http://gluetun-svc.gluetun.svc:8888', '--BASE_URL', url, '--pornstars', name,'--external_gallery', external_galleries] 
+       command=['bash', '/config/download.sh', '--http_proxy', 'http://gluetun-svc.gluetun.svc:8888', '--BASE_URL', url, '--pornstars', name]		
        template_manifest['spec']['template']['spec']['containers'][0]['command']=command
 
 
        #execute based on the value
        api_response = v1.create_namespaced_job(body=template_manifest,namespace='stash')
-    
     return HttpResponse(api_response)
 
 

django_code/stash_management/templates/album_form.html

@@ -0,0 +1,288 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+    <title>Stash form</title>
+    <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto|Varela+Round">
+    <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.5.0/css/bootstrap.min.css">
+    <link rel="stylesheet" href="https://fonts.googleapis.com/icon?family=Material+Icons">
+    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/font-awesome/4.7.0/css/font-awesome.min.css">
+
+    <style>
+        @charset "UTF-8";
+        .m-b-xs {
+            margin-bottom: 2%;
+        }
+
+        .m-b-md {
+            margin-bottom: 10%;
+        }
+
+        .m-t-xs {
+            margin-top: 2%;
+        }
+
+        .m-t-sm {
+            margin-top: 5%;
+        }
+
+        * {
+            outline: 0 !important;
+        }
+
+        body {
+            padding: 2% 0;
+            background-color: #37404a;
+        }
+
+        h1 {
+            font-family: "Roboto", sans-serif;
+            font-size: 2.2em;
+            font-weight: 300;
+            color: #85ffc7;
+            text-transform: uppercase;
+        }
+
+        p {
+            font-family: "Roboto", sans-serif;
+            font-size: 1.1em;
+            font-weight: 300;
+            color: #fff;
+        }
+
+        a {
+            color: #fff;
+        }
+        a:hover {
+            text-decoration: none;
+            color: #fff;
+        }
+
+        #survey-form label,
+        #survey-form .labels {
+            display: block;
+            margin-bottom: 2%;
+            font-family: "Roboto", sans-serif;
+            font-size: 1.1em;
+            font-weight: 300;
+            color: #fff;
+            letter-spacing: 0.5px;
+        }
+        #survey-form input::-webkit-input-placeholder {
+            color: transparent !important;
+        }
+        #survey-form input::-moz-placeholder {
+            color: transparent !important;
+        }
+        #survey-form input:-ms-input-placeholder {
+            color: transparent !important;
+        }
+        #survey-form input:-moz-placeholder {
+            color: transparent !important;
+        }
+        #survey-form input,
+        #survey-form select {
+            display: block;
+            width: 100%;
+            overflow: hidden;
+            outline: none;
+            border: 2px solid #333;
+        }
+        #survey-form input {
+            margin-top: 1.5%;
+            padding: 0 0 5px 0;
+            background: transparent;
+            border: none;
+            outline: none;
+            border-bottom: 2px solid #fff;
+            font-size: 1.1em;
+            font-weight: 300;
+            color: #85ffc7;
+        }
+        #survey-form input:focus {
+            border-color: #85ffc7;
+        }
+        #survey-form [type=checkbox],
+        #survey-form [type=radio] {
+            display: inline-block;
+            width: auto;
+            margin: 0px 10px 0 0;
+        }
+        #survey-form [type=checkbox],
+        #survey-form [type=radio] {
+            display: inline-block;
+            margin: 0 10px 0 0 !important;
+            position: relative;
+            top: 5px;
+            right: 0;
+            bottom: 0;
+            left: 0;
+            -webkit-appearance: none;
+            -moz-appearance: none;
+            -o-appearance: none;
+            appearance: none;
+            height: 23px;
+            width: 23px;
+            transition: all 0.15s ease-out 0s;
+            background: #37404a;
+            color: #fff;
+            cursor: pointer;
+            outline: none;
+            z-index: 1000;
+            border: 1px solid #fff;
+        }
+        #survey-form [type=checkbox]:hover,
+        #survey-form [type=radio]:hover {
+            border-color: #85ffc7;
+        }
+        #survey-form [type=checkbox]:checked:before,
+        #survey-form [type=radio]:checked:before {
+            display: inline-block;
+            height: 21px;
+            width: 21px;
+            position: relative;
+            left: 0;
+            bottom: 0;
+            content: "";
+            text-align: center;
+            font-family: "Glyphicons Halflings";
+            line-height: 20px;
+            font-size: 15px;
+            color: #85ffc7;
+        }
+        #survey-form [type=checkbox]:focus,
+        #survey-form [type=radio]:focus {
+            outline: none;
+            border-color: #fff;
+        }
+        #survey-form [type=radio] {
+            border-radius: 50%;
+        }
+        #survey-form [type=radio]:after {
+            border-radius: 50%;
+        }
+        #survey-form [type=number] {
+            width: 45px;
+        }
+        #survey-form [type=number]::-webkit-inner-spin-button, #survey-form [type=number]::-webkit-outer-spin-button {
+            -webkit-appearance: none;
+            -moz-appearance: none;
+            appearance: none;
+            margin: 0;
+        }
+        #survey-form select {
+            height: 40px;
+            padding-left: 5px;
+            background-color: #37404a;
+            border: 2px solid #fff;
+            color: #85ffc7;
+        }
+        #survey-form select option {
+            padding: 5px 10px;
+            font-weight: 300;
+            cursor: pointer;
+        }
+        #survey-form select option:hover {
+            background-color: #85ffc7;
+        }
+        #survey-form textarea {
+            resize: none;
+            margin-top: 2%;
+            padding: 10px 10px 0px 20px;
+            width: 100%;
+            height: 90px;
+            color: #85ffc7;
+            background-color: #37404a;
+            border: 2px solid #fff;
+        }
+        #survey-form .btn {
+            display: inline-block;
+            position: relative;
+            width: 100%;
+            margin: 3% 0 0 0;
+            height: 45px;
+            text-transform: uppercase;
+            text-decoration: none;
+            cursor: pointer;
+            border: 3px solid #85ffc7;
+            border-radius: 0;
+            font-weight: 500;
+            font-size: 1.2em;
+            color: #85ffc7;
+            text-align: center;
+            background: none;
+            transition: color 0.25s ease;
+        }
+        #survey-form .btn:after {
+            position: absolute;
+            content: "";
+            top: 0;
+            left: 0;
+            width: 0;
+            height: 100%;
+            background-color: #85ffc7;
+            transform-origin: left;
+            transition: width 0.5s ease;
+            z-index: -1;
+        }
+        #survey-form .btn:hover {
+            color: #37404a;
+        }
+        #survey-form .btn:hover:after {
+            width: 100%;
+        }
+        #survey-form fieldset {
+            margin: 5% 0 0 0;
+        }
+
+        .copyright {
+            text-align: center;
+            font-size: 10px;
+            color: #fff;
+        }
+        .copyright i {
+            color: #ef5350;
+        }
+
+        @media only screen and (max-width: 600px) {
+            body {
+                padding: 8% 0;
+            }
+        }
+    </style>
+</head>
+<main id="main" class="container">
+    <div class="row">
+        <div class="col-xs-12 col-lg-offset-3 col-lg-6">
+            <div class="m-b-md text-center">
+                <h1 id="title">Stash Form</h1>
+                <p id="description" class="description" class="text-center">Download form</p>
+            </div>
+                <form method="GET" action="/stash/album-form/" id="survey-form" name="survey-form">
+
+                <fieldset>
+                    <label for="name" id="name-label">
+                        Name *
+                        <input class="" type="text" id="name" name="name" placeholder="Enter your name (required)" required />
+                    </label>
+                </fieldset>
+                <fieldset>
+                    <label for="url" id="url-label">
+                        URL *
+                        <input class="" type="url" id="url" name="url" placeholder="Enter the URL (required)" required />
+                    </label>
+                </fieldset>
+
+                <fieldset>
+                    <label for="external_galleries">
+                        Any external galleries ?
+                        <textarea id="external_galleries" name="external_galleries" ></textarea>
+                    </label>
+                </fieldset>
+                <button id="submit" type="submit" class="btn">Submit the form</button>
+            </form>
+        </div>
+    </div>
+</main>
+</html>

django_code/stash_management/templates/incremental_form.html

@@ -0,0 +1,299 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+    <title>Stash form</title>
+    <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto|Varela+Round">
+    <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.5.0/css/bootstrap.min.css">
+    <link rel="stylesheet" href="https://fonts.googleapis.com/icon?family=Material+Icons">
+    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/font-awesome/4.7.0/css/font-awesome.min.css">
+
+    <style>
+        @charset "UTF-8";
+        .m-b-xs {
+            margin-bottom: 2%;
+        }
+
+        .m-b-md {
+            margin-bottom: 10%;
+        }
+
+        .m-t-xs {
+            margin-top: 2%;
+        }
+
+        .m-t-sm {
+            margin-top: 5%;
+        }
+
+        * {
+            outline: 0 !important;
+        }
+
+        body {
+            padding: 2% 0;
+            background-color: #37404a;
+        }
+
+        h1 {
+            font-family: "Roboto", sans-serif;
+            font-size: 2.2em;
+            font-weight: 300;
+            color: #85ffc7;
+            text-transform: uppercase;
+        }
+
+        p {
+            font-family: "Roboto", sans-serif;
+            font-size: 1.1em;
+            font-weight: 300;
+            color: #fff;
+        }
+
+        a {
+            color: #fff;
+        }
+        a:hover {
+            text-decoration: none;
+            color: #fff;
+        }
+
+        #survey-form label,
+        #survey-form .labels {
+            display: block;
+            margin-bottom: 2%;
+            font-family: "Roboto", sans-serif;
+            font-size: 1.1em;
+            font-weight: 300;
+            color: #fff;
+            letter-spacing: 0.5px;
+        }
+        #survey-form input::-webkit-input-placeholder {
+            color: transparent !important;
+        }
+        #survey-form input::-moz-placeholder {
+            color: transparent !important;
+        }
+        #survey-form input:-ms-input-placeholder {
+            color: transparent !important;
+        }
+        #survey-form input:-moz-placeholder {
+            color: transparent !important;
+        }
+        #survey-form input,
+        #survey-form select {
+            display: block;
+            width: 100%;
+            overflow: hidden;
+            outline: none;
+            border: 2px solid #333;
+        }
+        #survey-form input {
+            margin-top: 1.5%;
+            padding: 0 0 5px 0;
+            background: transparent;
+            border: none;
+            outline: none;
+            border-bottom: 2px solid #fff;
+            font-size: 1.1em;
+            font-weight: 300;
+            color: #85ffc7;
+        }
+        #survey-form input:focus {
+            border-color: #85ffc7;
+        }
+        #survey-form [type=checkbox],
+        #survey-form [type=radio] {
+            display: inline-block;
+            width: auto;
+            margin: 0px 10px 0 0;
+        }
+        #survey-form [type=checkbox],
+        #survey-form [type=radio] {
+            display: inline-block;
+            margin: 0 10px 0 0 !important;
+            position: relative;
+            top: 5px;
+            right: 0;
+            bottom: 0;
+            left: 0;
+            -webkit-appearance: none;
+            -moz-appearance: none;
+            -o-appearance: none;
+            appearance: none;
+            height: 23px;
+            width: 23px;
+            transition: all 0.15s ease-out 0s;
+            background: #37404a;
+            color: #fff;
+            cursor: pointer;
+            outline: none;
+            z-index: 1000;
+            border: 1px solid #fff;
+        }
+        #survey-form [type=checkbox]:hover,
+        #survey-form [type=radio]:hover {
+            border-color: #85ffc7;
+        }
+        #survey-form [type=checkbox]:checked:before,
+        #survey-form [type=radio]:checked:before {
+            display: inline-block;
+            height: 21px;
+            width: 21px;
+            position: relative;
+            left: 0;
+            bottom: 0;
+            content: "";
+            text-align: center;
+            font-family: "Glyphicons Halflings";
+            line-height: 20px;
+            font-size: 15px;
+            color: #85ffc7;
+        }
+        #survey-form [type=checkbox]:focus,
+        #survey-form [type=radio]:focus {
+            outline: none;
+            border-color: #fff;
+        }
+        #survey-form [type=radio] {
+            border-radius: 50%;
+        }
+        #survey-form [type=radio]:after {
+            border-radius: 50%;
+        }
+        #survey-form [type=number] {
+            width: 45px;
+        }
+        #survey-form [type=number]::-webkit-inner-spin-button, #survey-form [type=number]::-webkit-outer-spin-button {
+            -webkit-appearance: none;
+            -moz-appearance: none;
+            appearance: none;
+            margin: 0;
+        }
+        #survey-form select {
+            height: 40px;
+            padding-left: 5px;
+            background-color: #37404a;
+            border: 2px solid #fff;
+            color: #85ffc7;
+        }
+        #survey-form select option {
+            padding: 5px 10px;
+            font-weight: 300;
+            cursor: pointer;
+        }
+        #survey-form select option:hover {
+            background-color: #85ffc7;
+        }
+        #survey-form textarea {
+            resize: none;
+            margin-top: 2%;
+            padding: 10px 10px 0px 20px;
+            width: 100%;
+            height: 90px;
+            color: #85ffc7;
+            background-color: #37404a;
+            border: 2px solid #fff;
+        }
+        #survey-form .btn {
+            display: inline-block;
+            position: relative;
+            width: 100%;
+            margin: 3% 0 0 0;
+            height: 45px;
+            text-transform: uppercase;
+            text-decoration: none;
+            cursor: pointer;
+            border: 3px solid #85ffc7;
+            border-radius: 0;
+            font-weight: 500;
+            font-size: 1.2em;
+            color: #85ffc7;
+            text-align: center;
+            background: none;
+            transition: color 0.25s ease;
+        }
+        #survey-form .btn:after {
+            position: absolute;
+            content: "";
+            top: 0;
+            left: 0;
+            width: 0;
+            height: 100%;
+            background-color: #85ffc7;
+            transform-origin: left;
+            transition: width 0.5s ease;
+            z-index: -1;
+        }
+        #survey-form .btn:hover {
+            color: #37404a;
+        }
+        #survey-form .btn:hover:after {
+            width: 100%;
+        }
+        #survey-form fieldset {
+            margin: 5% 0 0 0;
+        }
+
+        .copyright {
+            text-align: center;
+            font-size: 10px;
+            color: #fff;
+        }
+        .copyright i {
+            color: #ef5350;
+        }
+
+        @media only screen and (max-width: 600px) {
+            body {
+                padding: 8% 0;
+            }
+        }
+    </style>
+</head>
+<main id="main" class="container">
+    <div class="row">
+        <div class="col-xs-12 col-lg-offset-3 col-lg-6">
+            <div class="m-b-md text-center">
+                <h1 id="title">Incremental Form</h1>
+                <p id="description" class="description" class="text-center">Download form</p>
+            </div>
+                <form method="GET" action="/stash/incremental-form/" id="survey-form" name="survey-form">
+
+                <fieldset>
+                    <label for="name" id="name-label">
+                        Name *
+                        <input class="" type="text" id="name" name="name" placeholder="Enter your name (required)" required />
+                    </label>
+                </fieldset>
+                <fieldset>
+                    <label for="url" id="url-label">
+                        URL of one image before the increment *
+                        <input class="" type="url" id="url" name="url" placeholder="Enter the URL before the increment(required)" required />
+                    </label>
+                </fieldset>
+                    <fieldset>
+                        <label for="url_suffix" id="url_suffix-label">
+                            suffix URL of one image after the increment *
+                            <input class="" type="string" id="url_suffix" name="url_suffix" placeholder="Enter the URL suffix (Example .png)(required)" required />
+                        </label>
+                    </fieldset>
+                    <fieldset>
+                        <label for="maximum_increment" id="maximum_increment_label">
+                            Maximum increment value *
+                            <input class="" type="text" id="maximum_increment" name="maximum_increment" placeholder="Enter the maximum increment(required)" required />
+                        </label>
+                    </fieldset>
+                    <fieldset>
+                        <label for="Minimum_increment" id="Minimum_increment_label">
+                            Minimum increment value 
+                            <input class="" type="text" id="Minimum_increment" name="Minimum_increment" placeholder="Enter the minimum increment (default 1)"  />
+                        </label>
+                    </fieldset>
+                <button id="submit" type="submit" class="btn">Submit the form</button>
+            </form>
+        </div>
+    </div>
+</main>
+</html>

django_code/stash_management/templates/stash_form.html

@@ -3,7 +3,7 @@
 <head>
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
-    <title>Scaled up</title>
+    <title>Stash form</title>
     <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto|Varela+Round">
     <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.5.0/css/bootstrap.min.css">
     <link rel="stylesheet" href="https://fonts.googleapis.com/icon?family=Material+Icons">
@@ -277,7 +277,7 @@
                 <fieldset>
                     <label for="external_galleries">
                         Any external galleries ?
-                        <textarea id="external_galleries" maxlength="194"></textarea>
+                        <textarea id="external_galleries" name="external_galleries" ></textarea>
                     </label>
                 </fieldset>
                 <button id="submit" type="submit" class="btn">Submit the form</button>

django_code/stash_management/views.py

@@ -5,4 +5,8 @@ from django.http import HttpResponse
 from django.shortcuts import render
 
 def hello(request):
-    return HttpResponse('<h1>Hello Django!</h1>')
+    return HttpResponse('<h1>Hello stash!</h1>')
+
+
+def home(request):
+    return render(request, 'listing.html')

docker-compose.yml

@@ -0,0 +1,11 @@
+services:
+  db:
+    image: django:local
+    restart: always
+    volumes:
+      - type: bind
+        target: /code
+        source: ./django_code
+    command: python manage.py runserver 0.0.0.0:7000
+    ports:
+      - 7000:7000

dockerfile

@@ -1,13 +1,12 @@
-FROM python:3
+FROM gitea.tryton.geeraert.eu/pi-ux-ce/modified-python:latest
+
+ENV PYTHONUNBUFFERED 1
 
 WORKDIR /code/
 
-COPY requirements.txt ./
+# Copy the rest of your application code (Changes frequently, invalidates subsequent cache)
-COPY django_code ./
+COPY . ./
-
-RUN --mount=type=cache,target=/root/.cache/pip pip install --upgrade pip
-RUN --mount=type=cache,target=/root/.cache/pip pip install --no-cache-dir -r requirements.txt
-COPY . .
 RUN python django_code/manage.py migrate
 
+# CMD is the command run when the container starts
 CMD [ "python","django_code/manage.py","runserver","0.0.0.0:8000" ]

download.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+
+#install required packages
+apk add curl wget
+
+#named argument generation
+skip_searching_gallery="${skip_searching_gallery:-false}"
+
+
+while [ $# -gt 0 ]; do
+    if [[ $1 == "--"* ]]; then
+        v="${1/--/}"
+        declare "$v"="$2"
+        shift
+    fi
+    shift
+done
+
+#check if we will use a HTTP proxy or not
+
+if [[ -z $http_proxy ]]; then
+        echo "not using proxy"
+        curl_command="curl"
+else
+        echo "using proxy"
+        curl_command="curl -x $http_proxy"
+
+fi
+
+
+#dowload from https://www.pornpics.com/pornstars/eiby-shine/
+DESTINATION_FOLDER="/data"
+PROXY_HTTP="http://gluetun-svc.gluetun.svc:8888"
+
+#BASE_URL="https://www.pornpics.com/pornstars/eiby-shine/"
+#pornstars="eiby-shine"
+
+#BASE_URL="https://www.pornpics.com/?q=shy+imp"
+#pornstars="shy-imp"
+
+#BASE_URL="https://www.pornpics.com/pornstars/gloria-sol/"
+#pornstars="gloria-sol"
+
+
+#BASE_URL="https://www.pornpics.com/pornstars/becky-summer"
+#pornstars="becky-summer"
+
+
+#clear files beforehand
+
+echo "Pornstar: "$pornstars
+echo "URL: "$BASE_URL
+
+mkdir -p $DESTINATION_FOLDER/$pornstars
+
+if [[ $skip_searching_gallery == true ]]; then
+        echo "not looping through galleries"
+		echo "using existing file instead"
+		#because we need to keep the file $DESTINATION_FOLDER/$pornstars-urls_of_galleries.txt with the existing URL , in this condition, 
+		#no deletion of this file
+                echo $external_gallery > $DESTINATION_FOLDER/$pornstars-urls_of_galleries.txt
+else
+        echo "looping through galleries"
+		#because we will re-generate this file, can be emptied
+		echo "" > $DESTINATION_FOLDER/$pornstars-urls_of_galleries.txt
+        $curl_command -s $BASE_URL |  grep -Eo "(http|https)://[a-zA-Z0-9./?=_%:-]*" | grep "https://www.pornpics.com/galleries/" |sort -u >> $DESTINATION_FOLDER/$pornstars-urls_of_galleries.txt
+fi
+
+echo "" > $DESTINATION_FOLDER/$pornstars-image_to_download.txt
+
+
+for f in $(cat $DESTINATION_FOLDER/$pornstars-urls_of_galleries.txt ); do
+	$curl_command $f | grep -Eo "(http|https)://[a-zA-Z0-9./?=_%:-]*" | grep "https://cdni.pornpics.com/1280" >> $DESTINATION_FOLDER/$pornstars-image_to_download.txt
+	echo $f
+done
+
+
+for g in $(cat $DESTINATION_FOLDER/$pornstars-image_to_download.txt ); do 
+	$curl_command -O --output-dir $DESTINATION_FOLDER/$pornstars $g
+#        wget -e use_proxy=yes -e http_proxy=$PROXY_HTTP $g -P $DESTINATION_FOLDER/$pornstars
+        echo $g 
+done
+exit 0

requirements.txt

@@ -8,3 +8,8 @@ ruamel.yaml
 pyyaml
 django-bootstrap-v5
 jinja2
+watchdog
+celery
+redis
+lxml
+requests