adding new form
Gitea Actions Demo / Explore-Gitea-Actions (push) Has been cancelled

This commit is contained in:
Pierre Geeraert
2025-07-29 09:39:41 +01:00
parent 2366d0c2a8
commit bccd344aa7
9310 changed files with 1330402 additions and 1 deletions
@@ -0,0 +1,3 @@
__author__ = "Oleg Butovich"
__copyright__ = "(c) Oleg Butovich 2013-2017"
__license__ = "MIT"
@@ -0,0 +1,171 @@
__author__ = "Oleg Butovich"
__copyright__ = "(c) Oleg Butovich 2013-2017"
__license__ = "MIT"
import json
import logging
import platform
import re
from itertools import chain
from shlex import split as shell_split
from proxmoxer.core import SERVICES
logger = logging.getLogger(__name__)
logger.setLevel(level=logging.WARNING)
try:
from shlex import join
def shell_join(args):
return join(args)
except ImportError:
from shlex import quote
def shell_join(args):
return " ".join([quote(arg) for arg in args])
class Response:
def __init__(self, content, status_code):
self.status_code = status_code
self.content = content
self.text = str(content)
self.headers = {"content-type": "application/json"}
def __str__(self):
return f"Response ({self.status_code}) {self.content}"
class CommandBaseSession:
def __init__(
self,
service="PVE",
timeout=5,
sudo=False,
):
self.service = service.lower()
self.timeout = timeout
self.sudo = sudo
def _exec(self, cmd):
raise NotImplementedError()
# noinspection PyUnusedLocal
def request(self, method, url, data=None, params=None, headers=None):
method = method.lower()
data = data or {}
params = params or {}
url = url.strip()
cmd = {"post": "create", "put": "set"}.get(method, method)
# separate out qemu exec commands to split into multiple argument pairs (issue#89)
data_command = None
if "/agent/exec" in url:
data_command = data.get("command")
if data_command is not None:
del data["command"]
# for 'upload' call some workaround
tmp_filename = ""
if url.endswith("upload"):
# copy file to temporary location on proxmox host
tmp_filename, _ = self._exec(
[
"python3",
"-c",
"import tempfile; import sys; tf = tempfile.NamedTemporaryFile(); sys.stdout.write(tf.name)",
]
)
tmp_filename = str(tmp_filename, "utf-8")
self.upload_file_obj(data["filename"], tmp_filename)
data["filename"] = data["filename"].name
data["tmpfilename"] = tmp_filename
command = [f"{self.service}sh", cmd, url]
# convert the options dict into a 2-tuple with the key formatted as a flag
option_pairs = []
for k, v in chain(data.items(), params.items()):
try:
option_pairs.append((f"-{k}", str(v, "utf-8")))
except TypeError:
option_pairs.append((f"-{k}", str(v)))
# add back in all the command arguments as their own pairs
if data_command is not None:
if isinstance(data_command, list):
command_arr = data_command
elif "Windows" not in platform.platform():
command_arr = shell_split(data_command)
for arg in command_arr:
option_pairs.append(("-command", arg))
# expand the list of 2-tuples into a flat list
options = [val for pair in option_pairs for val in pair]
additional_options = SERVICES[self.service.upper()].get("cli_additional_options", [])
full_cmd = command + options + additional_options
if self.sudo:
full_cmd = ["sudo"] + full_cmd
stdout, stderr = self._exec(full_cmd)
def is_http_status_string(s):
return re.match(r"\d\d\d [a-zA-Z]", str(s))
if stderr:
# assume if we got a task ID that the request was successful
task_id_pattern = re.compile(
r"UPID:[\w-]+:[0-9a-fA-F]{8}:[0-9a-fA-F]{8}:[0-9a-fA-F]{8}:\w+:[\w\._-]+:[\w\.@_-]+:\w*"
)
if task_id_pattern.search(str(stdout)) or task_id_pattern.search(str(stderr)):
status_code = 200
else:
# sometimes contains extra text like 'trying to acquire lock...OK'
status_code = next(
(
int(line.split()[0])
for line in stderr.splitlines()
if is_http_status_string(line)
),
500,
)
else:
status_code = 200
if stdout:
return Response(stdout, status_code)
return Response(stderr, status_code)
def upload_file_obj(self, file_obj, remote_path):
raise NotImplementedError()
class JsonSimpleSerializer:
def loads(self, response):
try:
return json.loads(response.content)
except (UnicodeDecodeError, ValueError):
return {"errors": response.content}
def loads_errors(self, response):
try:
return json.loads(response.text).get("errors")
except (UnicodeDecodeError, ValueError):
return {"errors": response.content}
class CommandBaseBackend:
def __init__(self):
self.session = None
self.target = None
def get_session(self):
return self.session
def get_base_url(self):
return ""
def get_serializer(self):
return JsonSimpleSerializer()
@@ -0,0 +1,386 @@
__author__ = "Oleg Butovich"
__copyright__ = "(c) Oleg Butovich 2013-2017"
__license__ = "MIT"
import io
import json
import logging
import os
import platform
import sys
import time
from shlex import split as shell_split
from proxmoxer.core import SERVICES, AuthenticationError, config_failure
logger = logging.getLogger(__name__)
logger.setLevel(level=logging.WARNING)
STREAMING_SIZE_THRESHOLD = 10 * 1024 * 1024 # 10 MiB
SSL_OVERFLOW_THRESHOLD = 2147483135 # 2^31 - 1 - 512
try:
import requests
from requests.auth import AuthBase
from requests.cookies import cookiejar_from_dict
# Disable warnings about using untrusted TLS
requests.packages.urllib3.disable_warnings()
except ImportError:
logger.error("Chosen backend requires 'requests' module\n")
sys.exit(1)
class ProxmoxHTTPAuthBase(AuthBase):
def __call__(self, req):
return req
def get_cookies(self):
return cookiejar_from_dict({})
def get_tokens(self):
return None, None
def __init__(self, timeout=5, service="PVE", verify_ssl=False, cert=None):
self.timeout = timeout
self.service = service
self.verify_ssl = verify_ssl
self.cert = cert
class ProxmoxHTTPAuth(ProxmoxHTTPAuthBase):
# number of seconds between renewing access tickets (must be less than 7200 to function correctly)
# if calls are made less frequently than 2 hrs, using the API token auth is recommended
renew_age = 3600
def __init__(self, username, password, otp=None, base_url="", **kwargs):
super().__init__(**kwargs)
self.base_url = base_url
self.username = username
self.pve_auth_ticket = ""
self._get_new_tokens(password=password, otp=otp)
def _get_new_tokens(self, password=None, otp=None):
if password is None:
# refresh from existing (unexpired) ticket
password = self.pve_auth_ticket
data = {"username": self.username, "password": password}
if otp:
data["otp"] = otp
response_data = requests.post(
self.base_url + "/access/ticket",
verify=self.verify_ssl,
timeout=self.timeout,
data=data,
cert=self.cert,
).json()["data"]
if response_data is None:
raise AuthenticationError(
"Couldn't authenticate user: {0} to {1}".format(
self.username, self.base_url + "/access/ticket"
)
)
if response_data.get("NeedTFA") is not None:
raise AuthenticationError(
"Couldn't authenticate user: missing Two Factor Authentication (TFA)"
)
self.birth_time = time.monotonic()
self.pve_auth_ticket = response_data["ticket"]
self.csrf_prevention_token = response_data["CSRFPreventionToken"]
def get_cookies(self):
return cookiejar_from_dict({self.service + "AuthCookie": self.pve_auth_ticket})
def get_tokens(self):
return self.pve_auth_ticket, self.csrf_prevention_token
def __call__(self, req):
# refresh ticket if older than `renew_age`
time_diff = time.monotonic() - self.birth_time
if time_diff >= self.renew_age:
logger.debug(f"refreshing ticket (age {time_diff})")
self._get_new_tokens()
# only attach CSRF token if needed (reduce interception risk)
if req.method != "GET":
req.headers["CSRFPreventionToken"] = self.csrf_prevention_token
return req
class ProxmoxHTTPApiTokenAuth(ProxmoxHTTPAuthBase):
def __init__(self, username, token_name, token_value, **kwargs):
super().__init__(**kwargs)
self.username = username
self.token_name = token_name
self.token_value = token_value
def __call__(self, req):
req.headers["Authorization"] = "{0}APIToken={1}!{2}{3}{4}".format(
self.service,
self.username,
self.token_name,
SERVICES[self.service]["token_separator"],
self.token_value,
)
req.cert = self.cert
return req
class JsonSerializer:
content_types = [
"application/json",
"application/x-javascript",
"text/javascript",
"text/x-javascript",
"text/x-json",
]
def get_accept_types(self):
return ", ".join(self.content_types)
def loads(self, response):
try:
return json.loads(response.content.decode("utf-8"))["data"]
except (UnicodeDecodeError, ValueError):
return {"errors": response.content}
def loads_errors(self, response):
try:
return json.loads(response.text).get("errors")
except (UnicodeDecodeError, ValueError):
return {"errors": response.content}
# pylint:disable=arguments-renamed
class ProxmoxHttpSession(requests.Session):
def request(
self,
method,
url,
params=None,
data=None,
headers=None,
cookies=None,
files=None,
auth=None,
timeout=None,
allow_redirects=True,
proxies=None,
hooks=None,
stream=None,
verify=None,
cert=None,
serializer=None,
):
a = auth or self.auth
c = cookies or self.cookies
# set verify flag from auth if request does not have this parameter explicitly
if verify is None:
verify = a.verify_ssl
if timeout is None:
timeout = a.timeout
# pull cookies from auth if not present
if (not c) and a:
cookies = a.get_cookies()
# filter out streams
files = files or {}
data = data or {}
total_file_size = 0
for k, v in data.copy().items():
# split qemu exec commands for proper parsing by PVE (issue#89)
if k == "command" and url.endswith("agent/exec"):
if isinstance(v, list):
data[k] = v
elif "Windows" not in platform.platform():
data[k] = shell_split(v)
if isinstance(v, io.IOBase):
total_file_size += get_file_size(v)
# add in filename from file pointer (patch for https://github.com/requests/toolbelt/pull/316)
# add Content-Type since Proxmox requires it (https://bugzilla.proxmox.com/show_bug.cgi?id=4344)
files[k] = (requests.utils.guess_filename(v), v, "application/octet-stream")
del data[k]
# if there are any large files, send all data and files using streaming multipart encoding
if total_file_size > STREAMING_SIZE_THRESHOLD:
try:
# pylint:disable=import-outside-toplevel
from requests_toolbelt import MultipartEncoder
encoder = MultipartEncoder(fields={**data, **files})
data = encoder
files = None
headers = {"Content-Type": encoder.content_type}
except ImportError:
# if the files will cause issues with the SSL 2GiB limit (https://bugs.python.org/issue42853#msg384566)
if total_file_size > SSL_OVERFLOW_THRESHOLD:
logger.warning(
"Install 'requests_toolbelt' to add support for files larger than 2GiB"
)
raise OverflowError("Unable to upload a payload larger than 2 GiB")
else:
logger.info(
"Installing 'requests_toolbelt' will decrease memory used during upload"
)
return super().request(
method,
url,
params,
data,
headers,
cookies,
files,
auth,
timeout,
allow_redirects,
proxies,
hooks,
stream,
verify,
cert,
)
class Backend:
def __init__(
self,
host,
user=None,
password=None,
otp=None,
port=None,
verify_ssl=True,
mode="json",
timeout=5,
token_name=None,
token_value=None,
path_prefix=None,
service="PVE",
cert=None,
):
self.cert = cert
host_port = ""
if len(host.split(":")) > 2: # IPv6
if host.startswith("["):
if "]:" in host:
host, host_port = host.rsplit(":", 1)
else:
host = f"[{host}]"
elif ":" in host:
host, host_port = host.split(":")
port = host_port if host_port.isdigit() else port
# if a port is not specified, use the default port for this service
if not port:
port = SERVICES[service]["default_port"]
self.mode = mode
if path_prefix is not None:
self.base_url = f"https://{host}:{port}/{path_prefix}/api2/{mode}"
else:
self.base_url = f"https://{host}:{port}/api2/{mode}"
if token_name is not None:
if "token" not in SERVICES[service]["supported_https_auths"]:
config_failure("{} does not support API Token authentication", service)
self.auth = ProxmoxHTTPApiTokenAuth(
user,
token_name,
token_value,
verify_ssl=verify_ssl,
timeout=timeout,
service=service,
cert=self.cert,
)
elif password is not None:
if "password" not in SERVICES[service]["supported_https_auths"]:
config_failure("{} does not support password authentication", service)
self.auth = ProxmoxHTTPAuth(
user,
password,
otp,
base_url=self.base_url,
verify_ssl=verify_ssl,
timeout=timeout,
service=service,
cert=self.cert,
)
else:
config_failure("No valid authentication credentials were supplied")
def get_session(self):
session = ProxmoxHttpSession()
session.cert = self.cert
session.auth = self.auth
# cookies are taken from the auth
session.headers["Connection"] = "keep-alive"
session.headers["accept"] = self.get_serializer().get_accept_types()
return session
def get_base_url(self):
return self.base_url
def get_serializer(self):
assert self.mode == "json"
return JsonSerializer()
def get_tokens(self):
"""Return the in-use auth and csrf tokens if using user/password auth."""
return self.auth.get_tokens()
def get_file_size(file_obj):
"""Returns the number of bytes in the given file object in total
file cursor remains at the same location as when passed in
:param fileObj: file object of which the get size
:type fileObj: file object
:return: total bytes in file object
:rtype: int
"""
# store existing file cursor location
starting_cursor = file_obj.tell()
# seek to end of file
file_obj.seek(0, os.SEEK_END)
size = file_obj.tell()
# reset cursor
file_obj.seek(starting_cursor)
return size
def get_file_size_partial(file_obj):
"""Returns the number of bytes in the given file object from the current cursor to the end
:param fileObj: file object of which the get size
:type fileObj: file object
:return: remaining bytes in file object
:rtype: int
"""
# store existing file cursor location
starting_cursor = file_obj.tell()
file_obj.seek(0, os.SEEK_END)
# get number of byte between where the cursor was set and the end
size = file_obj.tell() - starting_cursor
# reset cursor
file_obj.seek(starting_cursor)
return size
@@ -0,0 +1,25 @@
__author__ = "Markus Reiter"
__copyright__ = "(c) Markus Reiter 2022"
__license__ = "MIT"
import shutil
from subprocess import PIPE, Popen
from proxmoxer.backends.command_base import CommandBaseBackend, CommandBaseSession
class LocalSession(CommandBaseSession):
def _exec(self, cmd):
proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
stdout, stderr = proc.communicate(timeout=self.timeout)
return stdout.decode(), stderr.decode()
def upload_file_obj(self, file_obj, remote_path):
with open(remote_path, "wb") as dest_fp:
shutil.copyfileobj(file_obj, dest_fp)
class Backend(CommandBaseBackend):
def __init__(self, *args, **kwargs):
self.session = LocalSession(*args, **kwargs)
self.target = "localhost"
@@ -0,0 +1,67 @@
__author__ = "Oleg Butovich"
__copyright__ = "(c) Oleg Butovich 2013-2017"
__license__ = "MIT"
import logging
from proxmoxer.backends.command_base import (
CommandBaseBackend,
CommandBaseSession,
shell_join,
)
logger = logging.getLogger(__name__)
logger.setLevel(level=logging.WARNING)
try:
import openssh_wrapper
except ImportError:
import sys
logger.error("Chosen backend requires 'openssh_wrapper' module\n")
sys.exit(1)
class OpenSSHSession(CommandBaseSession):
def __init__(
self,
host,
user,
config_file=None,
port=22,
identity_file=None,
forward_ssh_agent=False,
**kwargs,
):
super().__init__(**kwargs)
self.host = host
self.user = user
self.config_file = config_file
self.port = port
self.forward_ssh_agent = forward_ssh_agent
self.identity_file = identity_file
self.ssh_client = self._connect()
def _connect(self):
return openssh_wrapper.SSHConnection(
self.host,
login=self.user,
port=str(self.port), # openssh_wrapper complains if this is an int
configfile=self.config_file,
identity_file=self.identity_file,
timeout=self.timeout,
)
def _exec(self, cmd):
ret = self.ssh_client.run(shell_join(cmd), forward_ssh_agent=self.forward_ssh_agent)
return ret.stdout, ret.stderr
def upload_file_obj(self, file_obj, remote_path):
self.ssh_client.scp((file_obj,), target=remote_path)
class Backend(CommandBaseBackend):
def __init__(self, *args, **kwargs):
self.session = OpenSSHSession(*args, **kwargs)
self.target = self.session.host
@@ -0,0 +1,77 @@
__author__ = "Oleg Butovich"
__copyright__ = "(c) Oleg Butovich 2013-2017"
__license__ = "MIT"
# spell-checker:ignore putfo
import logging
import os
from proxmoxer.backends.command_base import (
CommandBaseBackend,
CommandBaseSession,
shell_join,
)
logger = logging.getLogger(__name__)
logger.setLevel(level=logging.WARNING)
try:
import paramiko
except ImportError:
import sys
logger.error("Chosen backend requires 'paramiko' module\n")
sys.exit(1)
class SshParamikoSession(CommandBaseSession):
def __init__(self, host, user, password=None, private_key_file=None, port=22, **kwargs):
super().__init__(**kwargs)
self.host = host
self.user = user
self.password = password
self.private_key_file = private_key_file
self.port = port
self.ssh_client = self._connect()
def _connect(self):
ssh_client = paramiko.SSHClient()
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
if self.private_key_file:
key_filename = os.path.expanduser(self.private_key_file)
else:
key_filename = None
ssh_client.connect(
self.host,
username=self.user,
allow_agent=(not self.password),
look_for_keys=True,
key_filename=key_filename,
password=self.password,
timeout=self.timeout,
port=self.port,
)
return ssh_client
def _exec(self, cmd):
session = self.ssh_client.get_transport().open_session()
session.exec_command(shell_join(cmd))
stdout = session.makefile("rb", -1).read().decode()
stderr = session.makefile_stderr("rb", -1).read().decode()
return stdout, stderr
def upload_file_obj(self, file_obj, remote_path):
sftp = self.ssh_client.open_sftp()
sftp.putfo(file_obj, remote_path)
sftp.close()
class Backend(CommandBaseBackend):
def __init__(self, *args, **kwargs):
self.session = SshParamikoSession(*args, **kwargs)
self.target = self.session.host