from django.apps import AppConfig from django.http import HttpResponse from django.shortcuts import render from django.shortcuts import redirect from django.core.files import File from kubernetes import client, config from celery import shared_task from lxml import html import requests import os import json import time import random import mimetypes import ruamel.yaml import yaml from urllib import request as urlrequest from urllib import parse as urlparse proxy_url = 'http://gluetun-svc.gluetun.svc:8888' proxy = urlrequest.ProxyHandler({ 'http': proxy_url, 'https': proxy_url # Add the handler for HTTPS }) proxies = { "http" : proxy_url, "https" : proxy_url } destination_folder = "/stash/" def return_hello(): return "hellostash" def hello(request): return HttpResponse('

Hello stash!

') def form(request): return render(request, 'stash/stash_form.html',{}) def album(request): return render(request, 'stash/album_form.html',{}) def simple(request): return render(request, 'stash/simple_form.html',{}) def check_and_create_destination_folder(name): #check if the output directory exist and create if not if not os.path.isdir(destination_folder): os.makedirs(destination_folder) #check if the named directory exist and create if not destination_named_folder = os.path.join(destination_folder,name) if not os.path.isdir(destination_named_folder): os.makedirs(destination_named_folder) return destination_named_folder def album_form(request): name = request.GET.get('name') url = request.GET.get('url') external_galleries = request.GET.get('external_galleries') album_downloader.delay(name=name,url=url,external_galleries=external_galleries) text_generated = "successfully sent to the album downloader" return HttpResponse(text_generated, content_type="text/plain") @shared_task() def album_downloader(name,url,external_galleries): print(name+" "+url+" "+external_galleries) opener = urlrequest.build_opener(proxy) urlrequest.install_opener(opener) req = urlrequest.Request(url) urlcurl = urlrequest.urlopen(req).read() page = requests.get(url,proxies=proxies) webpage = html.fromstring(page.content) #create the prerequisites folder destination_named_folder = check_and_create_destination_folder(name=name) #get all URLs from the code source (searching for href tag on the code) #https://stackoverflow.com/questions/21455349/xpath-query-get-attribute-href-from-a-tag webpage_all_url = webpage.xpath('//a/@href') #search for the href that contains the galleries webpage_all_album_url = webpage.xpath("//a[contains(@href,'pornpics.com/galleries/')]/@href") #print(webpage_all_album_url[3]) #print(webpage_all_album_url) for album in webpage_all_album_url: print("Album "+album) print("") page = requests.get(album,proxies=proxies) webpage = html.fromstring(page.content) images_list = webpage.xpath("//a[contains(@href,'cdni.pornpics.com/1280/')]/@href") images_downloader.delay(name=name,images_url_list=images_list,destination_named_folder=destination_named_folder) print(images_list) return 0 @shared_task() def images_downloader(name,images_url_list,destination_named_folder): for image_url in images_url_list: split = urlparse.urlsplit(image_url) #filename = destination_folder + split.path.split("/")[-1] filename = os.path.join(destination_named_folder,split.path.split("/")[-1]) print("downloading: "+filename+" -- "+image_url) opener = urlrequest.build_opener(proxy) urlrequest.install_opener(opener) try: urlrequest.urlretrieve(image_url, filename ) except: print("Can't download "+filename+" -- "+image_url) individual_image_downloader.delay(image_url=image_url, filename=filename) return 0 @shared_task() def individual_image_downloader(image_url, filename ): print("Downloading again: "+filename+" -- "+image_url) opener = urlrequest.build_opener(proxy) urlrequest.install_opener(opener) try: urlrequest.urlretrieve(image_url, filename ) except: print("Still Can't download "+filename+" -- "+image_url+" == Cancelling") return 0 def simple_form(request): name = request.GET.get('name') reddit_downloader(username=name) text_generated = "successfully sent to the reddit downloader" return HttpResponse(text_generated, content_type="text/plain") def reddit_downloader(username): #create the prerequisites folder destination_named_folder = check_and_create_destination_folder(name=username) url = "https://www.reddit.com/user/"+username+"/submitted/.json?limit=1000" opener = urlrequest.build_opener(proxy) urlrequest.install_opener(opener) req = urlrequest.Request(url) urlcurl = urlrequest.urlopen(req).read() data = json.loads(urlcurl) picture_list=[] for children in data['data']['children']: print("adding to the list : "+children['data']['url']) picture_list.append(children['data']['url']) images_downloader.delay(name=username,images_url_list=picture_list,destination_named_folder=destination_named_folder) return 0 def redgif_donloader(username): #on te front page of the user , https://www.redgifs.com/users/princesspizookie # F12 and search network , find something that start as api.redgifs.com # and the file is search?order=new&count=40&type=g # copy as curl and you got the bearer #curl curl 'https://api.redgifs.com/v2/users/princesspizookie/search?order=new&count=40&type=g' #and you get the URLS , needs to be parsedw return 0 def incremental(request): return render(request, 'stash/incremental_form.html',{}) def incremental_form(request): name = request.GET.get('name') url = request.GET.get('url') url_suffix = request.GET.get('url_suffix') maximum_increment = int(request.GET.get('maximum_increment')) minimum_increment = request.GET.get('minimum_increment') increment_and_download.delay(minimum_increment=minimum_increment,maximum_increment=maximum_increment,url=url,url_suffix=url_suffix,name=name) text_generated = "successfully sent to celery" return HttpResponse(text_generated, content_type="text/plain") @shared_task() def increment_and_download(minimum_increment,maximum_increment,url,url_suffix,name): if minimum_increment == None: minimum_increment = 1 text_generated = "" minimum_increment = int(minimum_increment) maximum_increment = int(maximum_increment) #check if the output directory exist and create if not if not os.path.isdir(destination_folder): os.makedirs(destination_folder) #check if the named directory exist and create if not destination_named_folder = os.path.join(destination_folder,name) if not os.path.isdir(destination_named_folder): os.makedirs(destination_named_folder) print("\n downloading for "+name+" "+url+"XXX"+url_suffix) for increment in range(minimum_increment,maximum_increment): print("increment : "+str(increment)+"/"+str(maximum_increment)) if len(str(increment)) == 3: increment_generated ='0' + str(increment) elif len(str(increment)) == 2: increment_generated ='00' + str(increment) elif len(str(increment)) == 1: increment_generated ='000' + str(increment) url_generated = url+increment_generated+url_suffix print(url_generated) destination_filename = increment_generated+url_suffix destination_file = os.path.join(destination_named_folder,destination_filename) print(destination_file) opener = urlrequest.build_opener(proxy) urlrequest.install_opener(opener) urlrequest.urlretrieve(url_generated, destination_file ) text_generated = "\n downloading for "+name+" "+url+increment_generated+url_suffix print("download completed for "+name+" of "+str(maximum_increment)+" images") return text_generated def handle_survey_form(request): config.load_kube_config(config_file='/kubeconfig/config') v1 = client.BatchV1Api() name = request.GET.get('name') url = request.GET.get('url') external_galleries = str(request.GET.get('external_galleries')) stash_folder = "/stash" if external_galleries: print("external_galleries is not null") performer_external_galleries = stash_folder+"/"+name+"_external_galleries.txt" with open(performer_external_galleries, "a") as f: f.write(external_galleries) skip_searching_gallery = True else: print("external_galleries is null") skip_searching_gallery = False with open('django_code/stash_management/templates/template.stash-pornpics-dowloader.job.yml', 'r') as file: #get the manifest in a variable template_manifest from the manifest file template_manifest = yaml.safe_load(file) #edit the container name template_manifest['metadata']['name'] ="stash-pornpics-dowloader-"+name template_manifest['spec']['template']['spec']['containers'][0]['name'] ="stash-pornpics-dowloader-"+name #edit the command based on the input command=['bash', '/config/download.sh', '--http_proxy', 'http://gluetun-svc.gluetun.svc:8888', '--BASE_URL', url, '--pornstars', name] template_manifest['spec']['template']['spec']['containers'][0]['command']=command #execute based on the value api_response = v1.create_namespaced_job(body=template_manifest,namespace='stash') return HttpResponse(api_response)