This commit is contained in:
pierre geeraert
@@ -0,0 +1,276 @@
|
||||
from django.apps import AppConfig
|
||||
from django.http import HttpResponse
|
||||
from django.shortcuts import render
|
||||
from django.shortcuts import redirect
|
||||
from django.core.files import File
|
||||
from kubernetes import client, config
|
||||
from celery import shared_task
|
||||
|
||||
from lxml import html
|
||||
import requests
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
import random
|
||||
import mimetypes
|
||||
import ruamel.yaml
|
||||
import yaml
|
||||
from urllib import request as urlrequest
|
||||
from urllib import parse as urlparse
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
proxy_url = 'http://gluetun-svc.gluetun.svc:8888'
|
||||
proxy = urlrequest.ProxyHandler({
|
||||
'http': proxy_url,
|
||||
'https': proxy_url # Add the handler for HTTPS
|
||||
})
|
||||
|
||||
proxies = {
|
||||
"http" : proxy_url,
|
||||
"https" : proxy_url
|
||||
}
|
||||
|
||||
destination_folder = "/stash/"
|
||||
|
||||
def hello(request):
|
||||
return HttpResponse('<h1>Hello stash!</h1>')
|
||||
def form(request):
|
||||
return render(request, 'stash_form.html',{})
|
||||
def album(request):
|
||||
return render(request, 'album_form.html',{})
|
||||
def simple(request):
|
||||
return render(request, 'simple_form.html',{})
|
||||
def check_and_create_destination_folder(name):
|
||||
#check if the output directory exist and create if not
|
||||
if not os.path.isdir(destination_folder):
|
||||
os.makedirs(destination_folder)
|
||||
|
||||
#check if the named directory exist and create if not
|
||||
destination_named_folder = os.path.join(destination_folder,name)
|
||||
if not os.path.isdir(destination_named_folder):
|
||||
os.makedirs(destination_named_folder)
|
||||
|
||||
return destination_named_folder
|
||||
|
||||
def album_form(request):
|
||||
name = request.GET.get('name')
|
||||
url = request.GET.get('url')
|
||||
external_galleries = request.GET.get('external_galleries')
|
||||
|
||||
album_downloader.delay(name=name,url=url,external_galleries=external_galleries)
|
||||
|
||||
text_generated = "successfully sent to the album downloader"
|
||||
return HttpResponse(text_generated, content_type="text/plain")
|
||||
|
||||
@shared_task()
|
||||
def album_downloader(name,url,external_galleries):
|
||||
print(name+" "+url+" "+external_galleries)
|
||||
|
||||
opener = urlrequest.build_opener(proxy)
|
||||
urlrequest.install_opener(opener)
|
||||
|
||||
req = urlrequest.Request(url)
|
||||
urlcurl = urlrequest.urlopen(req).read()
|
||||
|
||||
page = requests.get(url,proxies=proxies)
|
||||
webpage = html.fromstring(page.content)
|
||||
|
||||
#create the prerequisites folder
|
||||
destination_named_folder = check_and_create_destination_folder(name=name)
|
||||
|
||||
|
||||
#get all URLs from the code source (searching for href tag on the code)
|
||||
#https://stackoverflow.com/questions/21455349/xpath-query-get-attribute-href-from-a-tag
|
||||
|
||||
webpage_all_url = webpage.xpath('//a/@href')
|
||||
|
||||
#search for the href that contains the galleries
|
||||
webpage_all_album_url = webpage.xpath("//a[contains(@href,'pornpics.com/galleries/')]/@href")
|
||||
#print(webpage_all_album_url[3])
|
||||
#print(webpage_all_album_url)
|
||||
|
||||
|
||||
for album in webpage_all_album_url:
|
||||
print("Album "+album)
|
||||
print("")
|
||||
page = requests.get(album,proxies=proxies)
|
||||
webpage = html.fromstring(page.content)
|
||||
|
||||
images_list = webpage.xpath("//a[contains(@href,'cdni.pornpics.com/1280/')]/@href")
|
||||
images_downloader.delay(name=name,images_url_list=images_list,destination_named_folder=destination_named_folder)
|
||||
print(images_list)
|
||||
return 0
|
||||
|
||||
@shared_task()
|
||||
def images_downloader(name,images_url_list,destination_named_folder):
|
||||
for image_url in images_url_list:
|
||||
split = urlparse.urlsplit(image_url)
|
||||
#filename = destination_folder + split.path.split("/")[-1]
|
||||
filename = os.path.join(destination_named_folder,split.path.split("/")[-1])
|
||||
|
||||
|
||||
print("downloading: "+filename+" -- "+image_url)
|
||||
|
||||
opener = urlrequest.build_opener(proxy)
|
||||
|
||||
urlrequest.install_opener(opener)
|
||||
try:
|
||||
urlrequest.urlretrieve(image_url, filename )
|
||||
except:
|
||||
print("Can't download "+filename+" -- "+image_url)
|
||||
individual_image_downloader.delay(image_url=image_url, filename=filename)
|
||||
|
||||
return 0
|
||||
|
||||
@shared_task()
|
||||
def individual_image_downloader(image_url, filename ):
|
||||
print("Downloading again: "+filename+" -- "+image_url)
|
||||
|
||||
opener = urlrequest.build_opener(proxy)
|
||||
|
||||
urlrequest.install_opener(opener)
|
||||
try:
|
||||
urlrequest.urlretrieve(image_url, filename )
|
||||
except:
|
||||
print("Still Can't download "+filename+" -- "+image_url+" == Cancelling")
|
||||
return 0
|
||||
|
||||
def simple_form(request):
|
||||
name = request.GET.get('name')
|
||||
reddit_downloader(username=name)
|
||||
|
||||
text_generated = "successfully sent to the reddit downloader"
|
||||
return HttpResponse(text_generated, content_type="text/plain")
|
||||
|
||||
def reddit_downloader(username):
|
||||
|
||||
#create the prerequisites folder
|
||||
destination_named_folder = check_and_create_destination_folder(name=username)
|
||||
|
||||
|
||||
|
||||
url = "https://www.reddit.com/user/"+username+"/submitted/.json?limit=1000"
|
||||
opener = urlrequest.build_opener(proxy)
|
||||
urlrequest.install_opener(opener)
|
||||
|
||||
req = urlrequest.Request(url)
|
||||
urlcurl = urlrequest.urlopen(req).read()
|
||||
data = json.loads(urlcurl)
|
||||
|
||||
picture_list=[]
|
||||
for children in data['data']['children']:
|
||||
print("adding to the list : "+children['data']['url'])
|
||||
picture_list.append(children['data']['url'])
|
||||
|
||||
images_downloader.delay(name=username,images_url_list=picture_list,destination_named_folder=destination_named_folder)
|
||||
|
||||
return 0
|
||||
|
||||
def redgif_donloader(username):
|
||||
#on te front page of the user , https://www.redgifs.com/users/princesspizookie
|
||||
# F12 and search network , find something that start as api.redgifs.com
|
||||
# and the file is search?order=new&count=40&type=g
|
||||
# copy as curl and you got the bearer
|
||||
|
||||
#curl curl 'https://api.redgifs.com/v2/users/princesspizookie/search?order=new&count=40&type=g'
|
||||
#and you get the URLS , needs to be parsedw
|
||||
return 0
|
||||
def incremental(request):
|
||||
return render(request, 'incremental_form.html',{})
|
||||
|
||||
def incremental_form(request):
|
||||
name = request.GET.get('name')
|
||||
url = request.GET.get('url')
|
||||
url_suffix = request.GET.get('url_suffix')
|
||||
maximum_increment = int(request.GET.get('maximum_increment'))
|
||||
minimum_increment = request.GET.get('minimum_increment')
|
||||
|
||||
increment_and_download.delay(minimum_increment=minimum_increment,maximum_increment=maximum_increment,url=url,url_suffix=url_suffix,name=name)
|
||||
|
||||
text_generated = "successfully sent to celery"
|
||||
return HttpResponse(text_generated, content_type="text/plain")
|
||||
|
||||
@shared_task()
|
||||
def increment_and_download(minimum_increment,maximum_increment,url,url_suffix,name):
|
||||
if minimum_increment == None:
|
||||
minimum_increment = 1
|
||||
|
||||
text_generated = ""
|
||||
minimum_increment = int(minimum_increment)
|
||||
maximum_increment = int(maximum_increment)
|
||||
|
||||
#check if the output directory exist and create if not
|
||||
if not os.path.isdir(destination_folder):
|
||||
os.makedirs(destination_folder)
|
||||
|
||||
#check if the named directory exist and create if not
|
||||
destination_named_folder = os.path.join(destination_folder,name)
|
||||
if not os.path.isdir(destination_named_folder):
|
||||
os.makedirs(destination_named_folder)
|
||||
|
||||
print("\n downloading for "+name+" "+url+"XXX"+url_suffix)
|
||||
|
||||
for increment in range(minimum_increment,maximum_increment):
|
||||
print("increment : "+str(increment)+"/"+str(maximum_increment))
|
||||
if len(str(increment)) == 3:
|
||||
increment_generated ='0' + str(increment)
|
||||
elif len(str(increment)) == 2:
|
||||
increment_generated ='00' + str(increment)
|
||||
elif len(str(increment)) == 1:
|
||||
increment_generated ='000' + str(increment)
|
||||
|
||||
url_generated = url+increment_generated+url_suffix
|
||||
print(url_generated)
|
||||
destination_filename = increment_generated+url_suffix
|
||||
destination_file = os.path.join(destination_named_folder,destination_filename)
|
||||
print(destination_file)
|
||||
opener = urlrequest.build_opener(proxy)
|
||||
urlrequest.install_opener(opener)
|
||||
urlrequest.urlretrieve(url_generated, destination_file )
|
||||
|
||||
text_generated = "\n downloading for "+name+" "+url+increment_generated+url_suffix
|
||||
print("download completed for "+name+" of "+str(maximum_increment)+" images")
|
||||
return text_generated
|
||||
def handle_survey_form(request):
|
||||
config.load_kube_config(config_file='/kubeconfig/config')
|
||||
v1 = client.BatchV1Api()
|
||||
name = request.GET.get('name')
|
||||
url = request.GET.get('url')
|
||||
external_galleries = str(request.GET.get('external_galleries'))
|
||||
|
||||
stash_folder = "/stash"
|
||||
|
||||
if external_galleries:
|
||||
print("external_galleries is not null")
|
||||
performer_external_galleries = stash_folder+"/"+name+"_external_galleries.txt"
|
||||
with open(performer_external_galleries, "a") as f:
|
||||
f.write(external_galleries)
|
||||
skip_searching_gallery = True
|
||||
else:
|
||||
print("external_galleries is null")
|
||||
skip_searching_gallery = False
|
||||
|
||||
with open('django_code/stash_management/templates/template.stash-pornpics-dowloader.job.yml', 'r') as file:
|
||||
#get the manifest in a variable template_manifest from the manifest file
|
||||
template_manifest = yaml.safe_load(file)
|
||||
|
||||
#edit the container name
|
||||
template_manifest['metadata']['name'] ="stash-pornpics-dowloader-"+name
|
||||
template_manifest['spec']['template']['spec']['containers'][0]['name'] ="stash-pornpics-dowloader-"+name
|
||||
|
||||
#edit the command based on the input
|
||||
command=['bash', '/config/download.sh', '--http_proxy', 'http://gluetun-svc.gluetun.svc:8888', '--BASE_URL', url, '--pornstars', name]
|
||||
template_manifest['spec']['template']['spec']['containers'][0]['command']=command
|
||||
|
||||
|
||||
#execute based on the value
|
||||
api_response = v1.create_namespaced_job(body=template_manifest,namespace='stash')
|
||||
return HttpResponse(api_response)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user