From a5fd5006df1e46d3d1c89e3ec17372246aabbd63 Mon Sep 17 00:00:00 2001 From: pierre Date: Thu, 10 Apr 2025 00:18:41 +0100 Subject: [PATCH] change the default name of a deploy, try to setup a job for stash but realised we don't have job yet on terraform and try to setup tika and gotenberg for paperless --- .../paperless/Paperless.secret.yaml | 13 ++++ external_manifests/stash/reddit_downloader.sh | 65 +++++++++++++++++ .../stash-reddit-dowloader.configmap.yml | 73 +++++++++++++++++++ .../stash/stash-reddit-dowloader.job.yml | 34 +++++++++ module.tf | 3 +- variables.auto.tfvars | 71 +++++++++++++----- 6 files changed, 239 insertions(+), 20 deletions(-) create mode 100644 external_manifests/paperless/Paperless.secret.yaml create mode 100755 external_manifests/stash/reddit_downloader.sh create mode 100644 external_manifests/stash/stash-reddit-dowloader.configmap.yml create mode 100644 external_manifests/stash/stash-reddit-dowloader.job.yml diff --git a/external_manifests/paperless/Paperless.secret.yaml b/external_manifests/paperless/Paperless.secret.yaml new file mode 100644 index 0000000..3f8d60a --- /dev/null +++ b/external_manifests/paperless/Paperless.secret.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +data: + POSTGRES_DB: cGFwZXJsZXNz + POSTGRES_USER: cGFwZXJsZXNz + POSTGRES_PASSWORD: cGFwZXJsZXNz + PAPERLESS_ADMIN_USER: YWRtaW4= + PAPERLESS_ADMIN_PASSWORD: YWRtaW4= + POSTGRES_HOST: cGFwZXItZGItc3Zj +kind: Secret +metadata: + name: paperless-credentials + namespace: paper +type: Opaque diff --git a/external_manifests/stash/reddit_downloader.sh b/external_manifests/stash/reddit_downloader.sh new file mode 100755 index 0000000..fd00a65 --- /dev/null +++ b/external_manifests/stash/reddit_downloader.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +# Check if username is provided +if [ -z "$1" ]; then + echo "Usage: $0 " + exit 1 +fi + +USER="$1" +DEST_FOLDER="${2:-./${USER}_images}" +USER_AGENT="Mozilla/5.0 (compatible; RedditImageDownloader/1.0)" + +mkdir -p "$DEST_FOLDER" + +AFTER="" +BASE_URL="https://www.reddit.com/user/$USER/submitted/.json" + +echo "Fetching all image posts from user $USER..." + +while true; do + # Build URL with pagination + if [ -z "$AFTER" ]; then + URL="${BASE_URL}?limit=100" + else + URL="${BASE_URL}?limit=100&after=${AFTER}" + fi + + # Fetch posts + RESPONSE=$(curl -s -A "$USER_AGENT" "$URL") + + # Validate JSON + if ! echo "$RESPONSE" | jq empty 2>/dev/null; then + echo "Error: Failed to fetch or parse JSON. Exiting." + break + fi + + # Extract image URLs + IMAGE_URLS=$(echo "$RESPONSE" | jq -r '.data.children[] | select(.data.url | test("\\.(jpg|jpeg|png|gif)$")) | .data.url') + + # Exit if no more images found + if [ -z "$IMAGE_URLS" ]; then + echo "No more images found on this page." + fi + + # Download images + echo "$IMAGE_URLS" | while read -r image_url; do + [ -z "$image_url" ] && continue + FILENAME=$(basename "$image_url") + if [ ! -f "$DEST_FOLDER/$FILENAME" ]; then + echo "Downloading $FILENAME..." + curl -s -A "$USER_AGENT" -o "$DEST_FOLDER/$FILENAME" "$image_url" + else + echo "Skipping existing file $FILENAME" + fi + done + + # Get 'after' token for pagination + AFTER=$(echo "$RESPONSE" | jq -r '.data.after') + if [ "$AFTER" == "null" ] || [ -z "$AFTER" ]; then + echo "Reached end of posts." + break + fi +done + +echo "Download complete. Images saved to $DEST_FOLDER" diff --git a/external_manifests/stash/stash-reddit-dowloader.configmap.yml b/external_manifests/stash/stash-reddit-dowloader.configmap.yml new file mode 100644 index 0000000..b168c59 --- /dev/null +++ b/external_manifests/stash/stash-reddit-dowloader.configmap.yml @@ -0,0 +1,73 @@ +apiVersion: v1 +data: + reddit_downloader.sh: | + #!/bin/bash + + # Check if username is provided + if [ -z "$1" ]; then + echo "Usage: $0 " + exit 1 + fi + + USER="$1" + DEST_FOLDER="${2:-./${USER}_images}" + USER_AGENT="Mozilla/5.0 (compatible; RedditImageDownloader/1.0)" + + mkdir -p "$DEST_FOLDER" + + AFTER="" + BASE_URL="https://www.reddit.com/user/$USER/submitted/.json" + + echo "Fetching all image posts from user $USER..." + + while true; do + # Build URL with pagination + if [ -z "$AFTER" ]; then + URL="${BASE_URL}?limit=100" + else + URL="${BASE_URL}?limit=100&after=${AFTER}" + fi + + # Fetch posts + RESPONSE=$(curl -s -A "$USER_AGENT" "$URL") + + # Validate JSON + if ! echo "$RESPONSE" | jq empty 2>/dev/null; then + echo "Error: Failed to fetch or parse JSON. Exiting." + break + fi + + # Extract image URLs + IMAGE_URLS=$(echo "$RESPONSE" | jq -r '.data.children[] | select(.data.url | test("\\.(jpg|jpeg|png|gif)$")) | .data.url') + + # Exit if no more images found + if [ -z "$IMAGE_URLS" ]; then + echo "No more images found on this page." + fi + + # Download images + echo "$IMAGE_URLS" | while read -r image_url; do + [ -z "$image_url" ] && continue + FILENAME=$(basename "$image_url") + if [ ! -f "$DEST_FOLDER/$FILENAME" ]; then + echo "Downloading $FILENAME..." + curl -s -A "$USER_AGENT" -o "$DEST_FOLDER/$FILENAME" "$image_url" + else + echo "Skipping existing file $FILENAME" + fi + done + + # Get 'after' token for pagination + AFTER=$(echo "$RESPONSE" | jq -r '.data.after') + if [ "$AFTER" == "null" ] || [ -z "$AFTER" ]; then + echo "Reached end of posts." + break + fi + done + + echo "Download complete. Images saved to $DEST_FOLDER" +kind: ConfigMap +metadata: + creationTimestamp: null + name: stash-reddit-dowloader + namespace: stash diff --git a/external_manifests/stash/stash-reddit-dowloader.job.yml b/external_manifests/stash/stash-reddit-dowloader.job.yml new file mode 100644 index 0000000..e502509 --- /dev/null +++ b/external_manifests/stash/stash-reddit-dowloader.job.yml @@ -0,0 +1,34 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: stash-reddit-dowloader + namespace: stash +spec: + template: + spec: + containers: + - name: stash-reddit-dowloader + image: alpine + imagePullPolicy: Always + volumeMounts: + - mountPath: /config + name: scripts + - mountPath: /data + mountPropagation: None + name: data-pvc + command: + - /bin/sh + - -c + - sleep 3600 +# - /config/reddit_downloader.sh Countem1a /data + restartPolicy: Never + volumes: + - configMap: + defaultMode: 365 + name: stash-reddit-dowloader + name: scripts + - name: data-pvc + persistentVolumeClaim: + claimName: data-pvc + backoffLimit: 4 + diff --git a/module.tf b/module.tf index 078369b..6f1b8b9 100644 --- a/module.tf +++ b/module.tf @@ -257,7 +257,8 @@ resource "kubernetes_deployment_v1" "automatic" { for_each = { for key, value in var.kubernetes : key => value if value.enabled == true } metadata { namespace = coalesce(each.value.namespace, each.key) - name = coalesce(each.value.namespace, each.key) + name = coalesce(each.value.name, each.key) + labels = { app = each.key deployment_method = "Terraform" diff --git a/variables.auto.tfvars b/variables.auto.tfvars index 700ff42..15e2acc 100644 --- a/variables.auto.tfvars +++ b/variables.auto.tfvars @@ -134,10 +134,11 @@ kubernetes = { stash = { enabled = false + replicas = 01 name = "stash" image = "stashapp/stash:latest" service_port = 9999 - + #extra_manifest = ["/opt/kubernetes/Terraform_kube/external_manifests/stash/stash-reddit-dowloader.configmap.yml"] environment = { STASH_STASH = { value = "/data/" @@ -347,7 +348,7 @@ kubernetes = { paper = { - enabled = false + enabled = false namespace = "paper" image = "ghcr.io/paperless-ngx/paperless-ngx:latest" service_port = 8000 @@ -361,14 +362,14 @@ kubernetes = { requests-cpu = "800Mi" requests-memory = "800Mi" } - secret_ref = ["secret1", "secret2"] - config_map_ref = ["config"] + extra_manifest = ["/opt/kubernetes/Terraform_kube/external_manifests/paperless/Paperless.secret.yaml"] + #config_map_ref = ["config"] environment = { POSTGRES_DB = { value_from = { secret_key_ref = { - name = "mysecret" + name = "paperless-credentials" key = "POSTGRES_DB" } } @@ -376,7 +377,7 @@ kubernetes = { POSTGRES_USER = { value_from = { secret_key_ref = { - name = "mysecret" + name = "paperless-credentials" key = "POSTGRES_USER" } } @@ -384,7 +385,7 @@ kubernetes = { POSTGRES_PASSWORD = { value_from = { secret_key_ref = { - name = "mysecret" + name = "paperless-credentials" key = "POSTGRES_PASSWORD" } } @@ -392,10 +393,21 @@ kubernetes = { PAPERLESS_URL = { value = "https://paper.tryton.geeraert.eu" } + PAPERLESS_TIKA_ENABLED = { + value = "1" + } + PAPERLESS_TIKA_GOTENBERG_ENDPOINT = { + value = "http://gotenberg-svc.gotenberg.svc:3000" + } + PAPERLESS_TIKA_ENDPOINT = { + value = "http://tika-svc.tika.svc:9998" + } + + PAPERLESS_DBHOST = { value_from = { secret_key_ref = { - name = "mysecret" + name = "paperless-credentials" key = "POSTGRES_HOST" } } @@ -408,7 +420,7 @@ kubernetes = { PAPERLESS_ADMIN_USER = { value_from = { secret_key_ref = { - name = "mysecret" + name = "paperless-credentials" key = "POSTGRES_USER" } } @@ -416,7 +428,7 @@ kubernetes = { PAPERLESS_ADMIN_PASSWORD = { value_from = { secret_key_ref = { - name = "mysecret" + name = "paperless-credentials" key = "POSTGRES_PASSWORD" } } @@ -440,15 +452,7 @@ kubernetes = { pvc_size = "2Mi" mount_path = "/usr/src/paperless/data" } - media = { - pvc = false - config_map = true - name = "gitea-runner-configfile" - pvc_size = "2Mi" - mount_path = "/tmp/usr/src/paperless/media" - } export = { - # pvc = true host_path = "/tmp" name = "export" pvc_size = "2Mi" @@ -462,10 +466,39 @@ kubernetes = { } } } + gotenberg = { + enabled = false + name = "gotenberg" + image = "gotenberg/gotenberg:7.9.2" + service_port = 3000 + + requirements = { + limits-cpu = "100m" + limits-memory = "300Mi" + requests-cpu = "50m" + requests-memory = "150Mi" + } + + + } + tika = { + enabled = false + name = "tika" + image = "ghcr.io/paperless-ngx/tika" + service_port = 9998 + + requirements = { + limits-cpu = "100m" + limits-memory = "300Mi" + requests-cpu = "50m" + requests-memory = "150Mi" + } + + + } - #} next = {