From 110cf01f75f95d6115f897e7ba4fe684508a0c96 Mon Sep 17 00:00:00 2001 From: John Turner Date: Wed, 10 Sep 2025 21:59:42 -0400 Subject: implement basic client --- pypaste/__init__.py | 208 ----------- pypaste/__main__.py | 162 --------- pypaste/client/__init__.py | 11 + pypaste/client/__main__.py | 72 ++++ pypaste/client/plugins/__init__.py | 11 + pypaste/client/plugins/zen/__init__.py | 636 +++++++++++++++++++++++++++++++++ pypaste/py.typed | 0 pypaste/s3/__init__.py | 126 ------- pypaste/s3/bucket.py | 146 -------- pypaste/server/__init__.py | 211 +++++++++++ pypaste/server/__main__.py | 163 +++++++++ pypaste/server/s3/__init__.py | 126 +++++++ pypaste/server/s3/bucket.py | 146 ++++++++ pypaste/server/sqlite/__init__.py | 103 ++++++ pypaste/sqlite/__init__.py | 103 ------ pypaste/sqlite/meson.build | 1 - 16 files changed, 1479 insertions(+), 746 deletions(-) delete mode 100644 pypaste/__main__.py create mode 100644 pypaste/client/__init__.py create mode 100644 pypaste/client/__main__.py create mode 100644 pypaste/client/plugins/__init__.py create mode 100644 pypaste/client/plugins/zen/__init__.py create mode 100644 pypaste/py.typed delete mode 100644 pypaste/s3/__init__.py delete mode 100644 pypaste/s3/bucket.py create mode 100644 pypaste/server/__init__.py create mode 100644 pypaste/server/__main__.py create mode 100644 pypaste/server/s3/__init__.py create mode 100644 pypaste/server/s3/bucket.py create mode 100644 pypaste/server/sqlite/__init__.py delete mode 100644 pypaste/sqlite/__init__.py delete mode 100644 pypaste/sqlite/meson.build diff --git a/pypaste/__init__.py b/pypaste/__init__.py index 0f0d5e2..c0b12d2 100644 --- a/pypaste/__init__.py +++ b/pypaste/__init__.py @@ -1,31 +1,5 @@ -# Copyright (C) 2025 John Turner - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - import sys -import asyncio -import secrets -import aiosqlite -from aiohttp import web from datetime import datetime -from dataclasses import dataclass -from typing import Optional, List, Tuple -from pygments import highlight -from pygments.lexers import guess_lexer, get_lexer_by_name -from pygments.formatters import HtmlFormatter -from pygments.styles import get_style_by_name -from abc import abstractmethod RESET = "\x1b[0m" RED = "\x1b[31m" @@ -46,185 +20,3 @@ def log_warning(msg: str) -> None: def log_error(msg: str) -> None: now = datetime.now().isoformat() print(f"{RED}[error]{RESET} {now} {msg}", file=sys.stderr) - - -def pygmentize( - content: str, syntax: Optional[str], style: str, line_numbers: str | bool -) -> str: - if syntax is not None: - try: - lexer = get_lexer_by_name(syntax) - except Exception: - log_warning(f"failed to find lexer for {syntax}") - lexer = guess_lexer(content) - else: - lexer = guess_lexer(content) - - try: - s = get_style_by_name(style) - except Exception as e: - log_warning(f"failed to find style: {style}: {e}") - s = get_style_by_name("default") - - formatter = HtmlFormatter(full=True, style=s, linenos="table") - - return highlight(content, lexer, formatter) - - -def generate_key(words: List[str], length: int) -> str: - choices = [] - for _ in range(length): - choices.append(secrets.choice(words)) - - return "-".join(word for word in choices).lower() - - -@dataclass -class Paste: - key: str - dt: datetime - syntax: Optional[str] - text: str - - -@dataclass -class Storage: - connection: aiosqlite.Connection - - @abstractmethod - async def setup(self) -> None: - pass - - @abstractmethod - async def insert(self, paste: Paste) -> None: - pass - - @abstractmethod - async def retrieve(self, key: str) -> Optional[Paste]: - pass - - @abstractmethod - async def delete(self, key) -> None: - pass - - @abstractmethod - async def vacuum(self, size: int) -> None: - pass - - async def read_row(self, key: str) -> Optional[Tuple[datetime, int, Optional[str]]]: - async with self.connection.execute( - "select pastes.datetime,pastes.size,pastes.syntax from pastes where pastes.key=? limit 1", - (key,), - ) as cursor: - match await cursor.fetchone(): - case [str(dt), int(size), syntax]: - return (datetime.fromisoformat(dt), size, syntax) - case None: - return None - case _: - raise Exception("unreachable") - - async def exists(self, key: str) -> bool: - async with self.connection.execute( - "select 1 from pastes where key=?", (key,) - ) as cursor: - return await cursor.fetchone() is not None - - -@dataclass -class AppConfig: - site: str - content_max_bytes: int - storage_max_bytes: int - key_length: int - dictionary: List[str] - default_style: str - line_numbers: str | bool - - -class App: - - def __init__(self, config: AppConfig, storage: Storage): - self.config = config - self.storage = storage - - async def download(self, request: web.Request) -> web.Response: - try: - key = request.match_info["key"] - except KeyError: - return web.HTTPBadRequest(text="provide a key to fetch") - - try: - paste = await self.storage.retrieve(key) - except Exception as e: - log_error(f"failed to retrieve paste {key}: {e}") - return web.HTTPInternalServerError() - - if paste is None: - log_info(f"{key} does not exist, returning 404") - return web.HTTPNotFound() - - if (syntax := request.query.get("syntax")) is None: - syntax = paste.syntax - - if (style := request.query.get("style")) is None: - style = self.config.default_style - - raw = request.query.get("raw") - - if raw is not None: - log_info(f"sending raw paste {key}") - - return web.HTTPOk(text=paste.text, content_type="text/plain") - else: - - def render(): - return pygmentize(paste.text, syntax, style, self.config.line_numbers) - - highlighted = await asyncio.to_thread(render) - - log_info( - f"sending rendered paste {key} with syntax {syntax} and style {style}" - ) - - return web.HTTPOk(text=highlighted, content_type="text/html") - - async def upload(self, request: web.Request) -> web.Response: - syntax = request.query.get("syntax") - - if ( - content_length := request.content_length - ) and content_length > self.config.content_max_bytes: - return web.HTTPBadRequest( - text=f"max content length is {self.config.content_max_bytes}" - ) - - try: - data = await request.read() - except Exception as e: - log_error(f"failed to read data: {e}") - return web.HTTPInternalServerError(text="failed to read data") - - try: - text = data.decode() - except UnicodeError: - return web.HTTPBadRequest( - text="content must be unicode only, no binary data is allowed" - ) - - key = generate_key(self.config.dictionary, self.config.key_length) - - try: - paste = Paste(key, datetime.now(), syntax, text) - await self.storage.insert(paste) - except Exception as e: - log_error(f"failed to insert paste {key} to storage: {e}") - return web.HTTPInternalServerError() - - url = f"{self.config.site}/paste/{key}" - - log_info( - f"uploaded paste {key} with syntax {syntax} of size {len(data)} bytes: {url}" - ) - - return web.HTTPOk(text=url) diff --git a/pypaste/__main__.py b/pypaste/__main__.py deleted file mode 100644 index 5215d15..0000000 --- a/pypaste/__main__.py +++ /dev/null @@ -1,162 +0,0 @@ -# Copyright (C) 2025 John Turner - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import sys -import os -import asyncio -import aiosqlite -from pypaste import App, AppConfig, Storage, log_error, log_info -from pypaste.s3 import S3 -from pypaste.sqlite import Sqlite -from socket import socket, AF_UNIX, SOCK_STREAM -from argparse import ArgumentParser -from aiohttp import web -from pathlib import Path - - -async def main() -> int: - parser = ArgumentParser() - subparsers = parser.add_subparsers(dest="command") - - parser.add_argument("--socket", required=True) - parser.add_argument("--socket-mode", default="0600") - parser.add_argument("--site", required=True) - parser.add_argument("--content-length-max-bytes", type=int, required=True) - parser.add_argument("--key-length", type=int, required=True) - parser.add_argument("--dictionary", type=Path, required=True) - parser.add_argument("--database", type=Path, required=True) - parser.add_argument("--storage-max-bytes", type=int, required=True) - parser.add_argument("--default-style", default="native") - parser.add_argument("--line-numbers", action="store_true") - parser.add_argument("--line-numbers-inline", action="store_true") - - s3parser = subparsers.add_parser("s3") - s3parser.add_argument("--endpoint", required=True) - s3parser.add_argument("--region", required=True) - s3parser.add_argument("--bucket", required=True) - s3parser.add_argument("--access-key", required=True) - s3parser.add_argument("--secret-key", type=Path, required=True) - - subparsers.add_parser("sqlite") - - args = parser.parse_args() - - try: - dictionary = args.dictionary.read_text().split("\n") - except Exception as e: - print(f"failed to open dictionary: {str(args.dictionary)}: {e}") - return 1 - - match [args.line_numbers, args.line_numbers_inline]: - case [True, _]: - line_numbers: str | bool = "table" - case [_, True]: - line_numbers = "inline" - case [_, _]: - line_numbers = False - - config = AppConfig( - args.site, - args.content_length_max_bytes, - args.storage_max_bytes, - args.key_length, - dictionary, - args.default_style, - line_numbers, - ) - - try: - connection = await aiosqlite.connect(args.database) - except Exception as e: - log_error(f"failed to connect to database {args.database}: {e}") - return 1 - - try: - await connection.execute( - ( - "create table if not exists pastes(key text, datetime text, size int, syntax text)" - ) - ) - await connection.commit() - except Exception as e: - log_error(f"failed to initialize database: {e}") - return 1 - - match args.command: - case "s3": - try: - secret_key = args.secret_key.read_text().strip() - except Exception as e: - log_error(f"failed to load secret key from {args.secret_key}: {e}") - return 1 - - storage: Storage = S3( - connection, - args.endpoint, - args.region, - args.bucket, - args.access_key, - secret_key, - ) - - await storage.setup() - case "sqlite": - storage = Sqlite(connection) - - await storage.setup() - - case _: - raise Exception(f"unsupported storage backend: {args.command}") - - pypaste = App(config, storage) - - app = web.Application() - - app.add_routes( - [web.get("/paste/{key}", pypaste.download), web.post("/paste", pypaste.upload)] - ) - - runner = web.AppRunner(app, logger=None) - - await runner.setup() - - sock = socket(AF_UNIX, SOCK_STREAM) - sock.bind(args.socket) - - os.chmod(args.socket, int(args.socket_mode, 8)) - - site = web.SockSite(runner, sock) - - await site.start() - - try: - await asyncio.Event().wait() - except asyncio.exceptions.CancelledError: - pass - finally: - os.remove(args.socket) - - return 0 - - -if __name__ == "__main__": - try: - log_info("starting pypaste") - sys.exit(asyncio.run(main())) - except KeyboardInterrupt: - sys.exit(0) - except Exception as e: - log_error(str(e)) - sys.exit(1) diff --git a/pypaste/client/__init__.py b/pypaste/client/__init__.py new file mode 100644 index 0000000..06b9100 --- /dev/null +++ b/pypaste/client/__init__.py @@ -0,0 +1,11 @@ +import io +from typing import Protocol, List, Optional, runtime_checkable + + +class PasteService(Protocol): + + def paste(self, buffer: io.BytesIO, syntax: Optional[str], raw: bool) -> str: + pass + + def supported_syntaxes(self) -> List[str]: + pass diff --git a/pypaste/client/__main__.py b/pypaste/client/__main__.py new file mode 100644 index 0000000..6bc2f14 --- /dev/null +++ b/pypaste/client/__main__.py @@ -0,0 +1,72 @@ +import sys +import pkgutil +import pypaste.client.plugins as plugins +from pypaste.client import PasteService +from importlib import import_module +from argparse import ArgumentParser +from pypaste import log_error +from typing import Dict + + +def register_services(): + modules = pkgutil.iter_modules(plugins.__path__, plugins.__name__ + ".") + + for _, name, _ in modules: + import_module(name) + + +def main() -> int: + parser = ArgumentParser() + subparsers = parser.add_subparsers(dest="command") + + list_services_parser = subparsers.add_parser("list-services") + + list_syntaxes_parser = subparsers.add_parser("list-syntaxes") + list_syntaxes_parser.add_argument("service") + + paste_parser = subparsers.add_parser("paste") + paste_parser.add_argument("service") + paste_parser.add_argument("--syntax") + paste_parser.add_argument("--raw") + + args = parser.parse_args() + + register_services() + + match args.command: + case "list-services": + for service in plugins.services.keys(): + print(service) + + return 0 + case "list-syntaxes": + service = plugins.services[args.service]() + + for syntax in service.supported_syntaxes(): + print(syntax) + + return 0 + case "paste": + service = plugins.services[args.service]() + + if ( + syntax := args.syntax + ) is not None and syntax not in service.supported_syntaxes(): + print(f"{syntax} is not supported for service {args.service}") + return 1 + + try: + text = sys.stdin.read() + except UnicodeError: + print("failed to decode text") + return 1 + + url = service.paste(text, syntax, args.raw) + + print(url) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pypaste/client/plugins/__init__.py b/pypaste/client/plugins/__init__.py new file mode 100644 index 0000000..018b673 --- /dev/null +++ b/pypaste/client/plugins/__init__.py @@ -0,0 +1,11 @@ +from pypaste.client import PasteService + +services = {} + + +def register_service(name): + + def register(plugin): + services[name] = plugin + + return register diff --git a/pypaste/client/plugins/zen/__init__.py b/pypaste/client/plugins/zen/__init__.py new file mode 100644 index 0000000..6ef8348 --- /dev/null +++ b/pypaste/client/plugins/zen/__init__.py @@ -0,0 +1,636 @@ +import io +import requests +from pypaste.client.plugins import register_service +from typing import Optional, List + +SYNTAXES = [ + "ABAP", + "AMDGPU", + "APL", + "ABNF", + "ActionScript 3", + "ActionScript", + "Ada", + "ADL", + "Agda", + "Aheui", + "Alloy", + "AmbientTalk", + "Ampl", + "HTML + Angular2", + "Angular2", + "ANTLR With ActionScript Target", + "ANTLR With C# Target", + "ANTLR With CPP Target", + "ANTLR With Java Target", + "ANTLR", + "ANTLR With ObjectiveC Target", + "ANTLR With Perl Target", + "ANTLR With Python Target", + "ANTLR With Ruby Target", + "ApacheConf", + "AppleScript", + "Arduino", + "Arrow", + "Arturo", + "ASCII armored", + "ASN.1", + "AspectJ", + "Asymptote", + "Augeas", + "AutoIt", + "autohotkey", + "Awk", + "BBC Basic", + "BBCode", + "BC", + "BQN", + "BST", + "BARE", + "Base Makefile", + "Bash", + "Bash Session", + "Batchfile", + "Bdd", + "Befunge", + "Berry", + "BibTeX", + "BlitzBasic", + "BlitzMax", + "Blueprint", + "BNF", + "Boa", + "Boo", + "Boogie", + "Brainfuck", + "BUGS", + "CAmkES", + "C", + "CMake", + "c-objdump", + "CPSA", + "CSS+UL4", + "aspx-cs", + "C#", + "ca65 assembler", + "cADL", + "CapDL", + "Cap'n Proto", + "Carbon", + "CBM BASIC V2", + "CDDL", + "Ceylon", + "CFEngine3", + "ChaiScript", + "Chapel", + "Charmci", + "HTML+Cheetah", + "JavaScript+Cheetah", + "Cheetah", + "XML+Cheetah", + "Cirru", + "Clay", + "Clean", + "Clojure", + "ClojureScript", + "COBOLFree", + "COBOL", + "CodeQL", + "CoffeeScript", + "Coldfusion CFC", + "Coldfusion HTML", + "cfstatement", + "COMAL-80", + "Common Lisp", + "Component Pascal", + "Coq", + "cplint", + "C++", + "cpp-objdump", + "Crmsh", + "Croc", + "Cryptol", + "Crystal", + "Csound Document", + "Csound Orchestra", + "Csound Score", + "CSS+Django/Jinja", + "CSS+Ruby", + "CSS+Genshi Text", + "CSS", + "CSS+PHP", + "CSS+Smarty", + "CUDA", + "Cypher", + "Cython", + "D", + "d-objdump", + "Darcs Patch", + "Dart", + "DASM16", + "Dax", + "Debian Control file", + "Debian Sources file", + "Delphi", + "Desktop file", + "Devicetree", + "dg", + "Diff", + "Django/Jinja", + "Zone", + "Docker", + "DTD", + "Duel", + "Dylan session", + "Dylan", + "DylanLID", + "ECL", + "eC", + "Earl Grey", + "Easytrieve", + "EBNF", + "Eiffel", + "Elixir iex session", + "Elixir", + "Elm", + "Elpi", + "EmacsLisp", + "E-mail", + "ERB", + "Erlang", + "Erlang erl session", + "HTML+Evoque", + "Evoque", + "XML+Evoque", + "execline", + "Ezhil", + "F#", + "FStar", + "Factor", + "Fancy", + "Fantom", + "Felix", + "Fennel", + "Fift", + "Fish", + "Flatline", + "FloScript", + "Forth", + "FortranFixed", + "Fortran", + "FoxPro", + "Freefem", + "FunC", + "Futhark", + "GAP session", + "GAP", + "GDScript", + "GLSL", + "GSQL", + "GAS", + "g-code", + "Genshi", + "Genshi Text", + "Gettext Catalog", + "Gherkin", + "Gleam", + "Gnuplot", + "Go", + "Golo", + "GoodData-CL", + "GoogleSQL", + "Gosu", + "Gosu Template", + "GraphQL", + "Graphviz", + "Groff", + "Groovy", + "HLSL", + "HTML+UL4", + "Haml", + "HTML+Handlebars", + "Handlebars", + "Hare", + "Haskell", + "Haxe", + "Hexdump", + "HSAIL", + "Hspec", + "HTML+Django/Jinja", + "HTML+Genshi", + "HTML", + "HTML+PHP", + "HTML+Smarty", + "HTTP", + "Hxml", + "Hy", + "Hybris", + "IDL", + "Icon", + "Idris", + "Igor", + "Inform 6", + "Inform 6 template", + "Inform 7", + "INI", + "Io", + "Ioke", + "IRC logs", + "Isabelle", + "J", + "JMESPath", + "JSLT", + "JAGS", + "Janet", + "Jasmin", + "Java", + "JavaScript+Django/Jinja", + "JavaScript+Ruby", + "JavaScript+Genshi Text", + "JavaScript", + "JavaScript+PHP", + "JavaScript+Smarty", + "Javascript+UL4", + "JCL", + "JSGF", + "JSON5", + "JSONBareObject", + "JSON-LD", + "JSON", + "Jsonnet", + "Java Server Page", + "JSX", + "Julia console", + "Julia", + "Juttle", + "K", + "Kal", + "Kconfig", + "Kernel log", + "Koka", + "Kotlin", + "Kuin", + "Kusto", + "LSL", + "CSS+Lasso", + "HTML+Lasso", + "JavaScript+Lasso", + "Lasso", + "XML+Lasso", + "LDAP configuration file", + "LDIF", + "Lean", + "Lean4", + "LessCss", + "Lighttpd configuration file", + "LilyPond", + "Limbo", + "liquid", + "Literate Agda", + "Literate Cryptol", + "Literate Haskell", + "Literate Idris", + "LiveScript", + "LLVM", + "LLVM-MIR Body", + "LLVM-MIR", + "Logos", + "Logtalk", + "Lua", + "Luau", + "MCFunction", + "MCSchema", + "MIME", + "MIPS", + "MOOCode", + "MSDOS Session", + "Macaulay2", + "Makefile", + "CSS+Mako", + "HTML+Mako", + "JavaScript+Mako", + "Mako", + "XML+Mako", + "Maple", + "MAQL", + "Markdown", + "Mask", + "Mason", + "Mathematica", + "Matlab", + "Matlab session", + "Maxima", + "Meson", + "MiniD", + "MiniScript", + "Modelica", + "Modula-2", + "MoinMoin/Trac Wiki markup", + "Mojo", + "Monkey", + "Monte", + "MoonScript", + "Mosel", + "CSS+mozpreproc", + "mozhashpreproc", + "Javascript+mozpreproc", + "mozpercentpreproc", + "XUL+mozpreproc", + "MQL", + "Mscgen", + "MuPAD", + "MXML", + "MySQL", + "CSS+Myghty", + "HTML+Myghty", + "JavaScript+Myghty", + "Myghty", + "XML+Myghty", + "NCL", + "NSIS", + "NASM", + "objdump-nasm", + "Nemerle", + "nesC", + "NestedText", + "NewLisp", + "Newspeak", + "Nginx configuration file", + "Nimrod", + "Nit", + "Nix", + "Node.js REPL console session", + "Notmuch", + "NuSMV", + "NumPy", + "Numba_IR", + "objdump", + "Objective-C", + "Objective-C++", + "Objective-J", + "OCaml", + "Octave", + "ODIN", + "OMG Interface Definition Language", + "Ooc", + "Opa", + "OpenEdge ABL", + "OpenSCAD", + "Org Mode", + "Text output", + "PacmanConf", + "Pan", + "ParaSail", + "Pawn", + "PDDL", + "PEG", + "Perl6", + "Perl", + "Phix", + "PHP", + "Pig", + "Pike", + "PkgConfig", + "PL/pgSQL", + "Pointless", + "Pony", + "Portugol", + "PostScript", + "PostgreSQL console (psql)", + "PostgreSQL EXPLAIN dialect", + "PostgreSQL SQL dialect", + "POVRay", + "PowerShell", + "PowerShell Session", + "Praat", + "Procfile", + "Prolog", + "PromQL", + "Promela", + "Properties", + "Protocol Buffer", + "PRQL", + "PsySH console session for PHP", + "PTX", + "Pug", + "Puppet", + "PyPy Log", + "Python 2.x", + "Python 2.x Traceback", + "Python console session", + "Python", + "Python Traceback", + "Python+UL4", + "QBasic", + "Q", + "QVTO", + "Qlik", + "QML", + "RConsole", + "Relax-NG Compact", + "RPMSpec", + "Racket", + "Ragel in C Host", + "Ragel in CPP Host", + "Ragel in D Host", + "Embedded Ragel", + "Ragel in Java Host", + "Ragel", + "Ragel in Objective C Host", + "Ragel in Ruby Host", + "Raw token data", + "Rd", + "ReasonML", + "REBOL", + "Red", + "Redcode", + "reg", + "Rego", + "ResourceBundle", + "Rexx", + "RHTML", + "Ride", + "Rita", + "Roboconf Graph", + "Roboconf Instances", + "RobotFramework", + "RQL", + "RSL", + "reStructuredText", + "TrafficScript", + "Ruby irb session", + "Ruby", + "Rust", + "SAS", + "S", + "Standard ML", + "SNBT", + "SARL", + "Sass", + "Savi", + "Scala", + "Scaml", + "scdoc", + "Scheme", + "Scilab", + "SCSS", + "Sed", + "ShExC", + "Shen", + "Sieve", + "Silver", + "Singularity", + "Slash", + "Slim", + "Slurm", + "Smali", + "Smalltalk", + "SmartGameFormat", + "Smarty", + "Smithy", + "Snobol", + "Snowball", + "Solidity", + "Soong", + "Sophia", + "SourcePawn", + "Debian Sourcelist", + "SPARQL", + "Spice", + "SQL+Jinja", + "SQL", + "sqlite3con", + "SquidConf", + "Srcinfo", + "Scalate Server Page", + "Stan", + "Stata", + "SuperCollider", + "Swift", + "SWIG", + "systemverilog", + "Systemd", + "TAP", + "Typographic Number Theory", + "TOML", + "TableGen", + "Tact", + "TADS 3", + "Tal", + "TASM", + "Tcl", + "Tcsh", + "Tcsh Session", + "Tea", + "teal", + "Tera Term macro", + "Termcap", + "Terminfo", + "Terraform", + "TeX", + "Text only", + "ThingsDB", + "Thrift", + "tiddler", + "Tl-b", + "TLS Presentation Language", + "Todotxt", + "Transact-SQL", + "Treetop", + "TSX", + "Turtle", + "HTML+Twig", + "Twig", + "TypeScript", + "TypoScriptCssData", + "TypoScriptHtmlData", + "TypoScript", + "Typst", + "UL4", + "ucode", + "Unicon", + "Unix/Linux config files", + "UrbiScript", + "urlencoded", + "USD", + "VBScript", + "VCL", + "VCLSnippets", + "VCTreeStatus", + "VGL", + "Vala", + "aspx-vb", + "VB.net", + "HTML+Velocity", + "Velocity", + "XML+Velocity", + "Verifpal", + "verilog", + "vhdl", + "VimL", + "Visual Prolog Grammar", + "Visual Prolog", + "Vue", + "Vyper", + "WDiff", + "WebAssembly", + "Web IDL", + "WebGPU Shading Language", + "Whiley", + "Wikitext", + "World of Warcraft TOC", + "Wren", + "X10", + "XML+UL4", + "XQuery", + "XML+Django/Jinja", + "XML+Ruby", + "XML", + "XML+PHP", + "XML+Smarty", + "Xorg", + "X++", + "XSLT", + "Xtend", + "xtlang", + "YAML+Jinja", + "YAML", + "YANG", + "YARA", + "Zeek", + "Zephir", + "Zig", + "ANSYS parametric design language", + "CSS+Mako", + "HTML+Mako", + "JavaScript+Mako", + "Mako", + "XML+Mako", +] + +URL: str = "https://zen.jturnerusa.dev/paste" + + +@register_service("zen") +class Zen: + + def paste(self, buffer: io.BytesIO, syntax: Optional[str], raw: bool) -> str: + params = {} + + if syntax is not None: + params["syntax"] = syntax + + if raw: + params["raw"] = "true" + + req = requests.post(URL, params=params, data=buffer) + + if req.status_code != 200: + raise Exception( + f"failed to paste to pypaste with status code: {req.status_code}" + ) + + return req.text + + def supported_syntaxes(self) -> List[str]: + return SYNTAXES diff --git a/pypaste/py.typed b/pypaste/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/pypaste/s3/__init__.py b/pypaste/s3/__init__.py deleted file mode 100644 index 7d21703..0000000 --- a/pypaste/s3/__init__.py +++ /dev/null @@ -1,126 +0,0 @@ -# Copyright (C) 2025 John Turner - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import asyncio -import zstandard -import asyncio -import aiosqlite -from pypaste import Storage, Paste -from pypaste.s3.bucket import Bucket -from dataclasses import dataclass -from typing import Optional - - -@dataclass -class S3(Storage): - connection: aiosqlite.Connection - - def __init__( - self, - connection: aiosqlite.Connection, - endpoint: str, - region: str, - bucket: str, - access_key: str, - secret_key: str, - ): - self.connection = connection - self.bucket = Bucket(endpoint, region, bucket, access_key, secret_key) - - async def setup(self) -> None: - await self.connection.execute("create table if not exists s3(key text)") - await self.connection.commit() - - async def insert(self, paste: Paste) -> None: - def compress(): - return zstandard.compress(paste.text.encode()) - - compressed = await asyncio.to_thread(compress) - - await self.connection.execute( - "insert into pastes values(?, ?, ?, ?)", - (paste.key, paste.dt.isoformat(), len(compressed), paste.syntax), - ) - - try: - await self.bucket.put(paste.key, compressed) - await self.connection.commit() - except Exception as e: - await self.connection.rollback() - raise e - - async def retrieve(self, key: str) -> Optional[Paste]: - if not await self.exists(key): - return None - - row = await self.read_row(key) - - assert row is not None - - (dt, size, syntax) = row - - data = await self.bucket.get(key) - - assert data is not None - - def decompress() -> str: - return zstandard.decompress(data).decode() - - text = await asyncio.to_thread(decompress) - - return Paste(key, dt, syntax, text) - - async def delete(self, key: str) -> None: - await self.connection.execute("delete from pastes where key=?", (key,)) - - try: - await self.bucket.delete(key) - await self.connection.commit() - except Exception as e: - await self.connection.rollback() - raise e - - async def vacuum(self, max: int) -> None: - while True: - async with self.connection.execute( - ( - "select sum(pastes.size) from pastes " - "inner join s3 on s3.key " - "where s3.key=pastes.key" - ) - ) as cursor: - if (row := await cursor.fetchone()) is None: - return - else: - use = row[0] - - async with self.connection.execute( - ( - "select pastes.key from pastes " - "inner join s3 on s3.key " - "where s3.key=pastes.key " - "order by pastes.datetime " - "limit 1" - ) - ) as cursor: - if (row := await cursor.fetchone()) is None: - return - else: - oldest = row[0] - - if use > max: - await self.delete(oldest) - else: - return diff --git a/pypaste/s3/bucket.py b/pypaste/s3/bucket.py deleted file mode 100644 index c795bbd..0000000 --- a/pypaste/s3/bucket.py +++ /dev/null @@ -1,146 +0,0 @@ -# Copyright (C) 2025 John Turner - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import aiohttp -from datetime import datetime, UTC -from typing import Dict -from dataclasses import dataclass -from bozo4 import s3v4_sign_request, s3v4_datetime_string -from hashlib import sha256 -from typing import Optional - - -@dataclass -class Request: - url: str - headers: Dict[str, str] - - -@dataclass -class Bucket: - endpoint: str - region: str - bucket: str - access_key: str - secret_key: str - - async def get(self, key: str) -> Optional[bytes]: - now = datetime.now(UTC) - - headers = { - "Host": self.endpoint, - "X-Amz-Date": s3v4_datetime_string(now), - "X-Amz-Content-SHA256": "UNSIGNED-PAYLOAD", - } - - auth = s3v4_sign_request( - endpoint=self.endpoint, - region=self.region, - access_key=self.access_key, - secret_key=self.secret_key, - request_method="GET", - date=now, - payload_hash="UNSIGNED-PAYLOAD", - uri=f"/{self.bucket}/{key}", - parameters={}, - headers=headers, - service="s3", - ) - - headers["Authorization"] = auth - - url = f"https://{self.endpoint}/{self.bucket}/{key}" - - async with aiohttp.ClientSession().get(url, headers=headers) as get: - match get.status: - case 200: - return await get.read() - case 404: - return None - case _: - raise Exception( - f"failed to get {self.endpoint}/{self.bucket}/{key} with status {get.status}" - ) - - async def put(self, key: str, data: bytes) -> None: - payload_hash = sha256(data).hexdigest() - - now = datetime.now(UTC) - - headers = { - "Host": self.endpoint, - "X-Amz-Date": s3v4_datetime_string(now), - "X-Amz-Content-SHA256": payload_hash, - "Content-Length": str(len(data)), - "Content-Type": "application/octect-stream", - } - - auth = s3v4_sign_request( - endpoint=self.endpoint, - region=self.region, - access_key=self.access_key, - secret_key=self.secret_key, - request_method="PUT", - date=now, - payload_hash=payload_hash, - uri=f"/{self.bucket}/{key}", - parameters={}, - headers=headers, - service="s3", - ) - - headers["Authorization"] = auth - - url = f"https://{self.endpoint}/{self.bucket}/{key}" - - async with aiohttp.ClientSession().put(url, headers=headers, data=data) as put: - if put.status != 200: - raise Exception( - f"failed put {self.endpoint}/{self.bucket}/{key} with {put.status}" - ) - - async def delete(self, key: str) -> None: - now = datetime.now(UTC) - - headers = { - "Host": self.endpoint, - "X-Amz-Date": s3v4_datetime_string(now), - "X-Amz-Content-SHA256": "UNSIGNED-PAYLOAD", - "Content-Length": "0", - } - - auth = s3v4_sign_request( - endpoint=self.endpoint, - region=self.region, - access_key=self.access_key, - secret_key=self.secret_key, - request_method="DELETE", - date=now, - payload_hash="UNSIGNED-PAYLOAD", - uri=f"/{self.bucket}/{key}", - parameters={}, - headers=headers, - service="s3", - ) - - headers["Authorization"] = auth - - url = f"https://{self.endpoint}/{self.bucket}/{key}" - - async with aiohttp.ClientSession().delete(url, headers=headers) as delete: - if delete.status != 200: - raise Exception( - f"failed to delete {self.endpoint}/{self.bucket}/{key} with {delete.status}" - ) diff --git a/pypaste/server/__init__.py b/pypaste/server/__init__.py new file mode 100644 index 0000000..ad07ecd --- /dev/null +++ b/pypaste/server/__init__.py @@ -0,0 +1,211 @@ +# Copyright (C) 2025 John Turner + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import sys +import asyncio +import secrets +import aiosqlite +from aiohttp import web +from datetime import datetime +from dataclasses import dataclass +from typing import Optional, List, Tuple +from pypaste import log_error, log_warning, log_info +from pygments import highlight +from pygments.lexers import guess_lexer, get_lexer_by_name +from pygments.formatters import HtmlFormatter +from pygments.styles import get_style_by_name +from abc import abstractmethod + + +def pygmentize( + content: str, syntax: Optional[str], style: str, line_numbers: str | bool +) -> str: + if syntax is not None: + try: + lexer = get_lexer_by_name(syntax) + except Exception: + log_warning(f"failed to find lexer for {syntax}") + lexer = guess_lexer(content) + else: + lexer = guess_lexer(content) + + try: + s = get_style_by_name(style) + except Exception as e: + log_warning(f"failed to find style: {style}: {e}") + s = get_style_by_name("default") + + formatter = HtmlFormatter(full=True, style=s, linenos="table") + + return highlight(content, lexer, formatter) + + +def generate_key(words: List[str], length: int) -> str: + choices = [] + for _ in range(length): + choices.append(secrets.choice(words)) + + return "-".join(word for word in choices).lower() + + +@dataclass +class Paste: + key: str + dt: datetime + syntax: Optional[str] + text: str + + +@dataclass +class Storage: + connection: aiosqlite.Connection + + @abstractmethod + async def setup(self) -> None: + pass + + @abstractmethod + async def insert(self, paste: Paste) -> None: + pass + + @abstractmethod + async def retrieve(self, key: str) -> Optional[Paste]: + pass + + @abstractmethod + async def delete(self, key) -> None: + pass + + @abstractmethod + async def vacuum(self, size: int) -> None: + pass + + async def read_row(self, key: str) -> Optional[Tuple[datetime, int, Optional[str]]]: + async with self.connection.execute( + "select pastes.datetime,pastes.size,pastes.syntax from pastes where pastes.key=? limit 1", + (key,), + ) as cursor: + match await cursor.fetchone(): + case [str(dt), int(size), syntax]: + return (datetime.fromisoformat(dt), size, syntax) + case None: + return None + case _: + raise Exception("unreachable") + + async def exists(self, key: str) -> bool: + async with self.connection.execute( + "select 1 from pastes where key=?", (key,) + ) as cursor: + return await cursor.fetchone() is not None + + +@dataclass +class AppConfig: + site: str + content_max_bytes: int + storage_max_bytes: int + key_length: int + dictionary: List[str] + default_style: str + line_numbers: str | bool + + +class App: + + def __init__(self, config: AppConfig, storage: Storage): + self.config = config + self.storage = storage + + async def download(self, request: web.Request) -> web.Response: + try: + key = request.match_info["key"] + except KeyError: + return web.HTTPBadRequest(text="provide a key to fetch") + + try: + paste = await self.storage.retrieve(key) + except Exception as e: + log_error(f"failed to retrieve paste {key}: {e}") + return web.HTTPInternalServerError() + + if paste is None: + log_info(f"{key} does not exist, returning 404") + return web.HTTPNotFound() + + if (syntax := request.query.get("syntax")) is None: + syntax = paste.syntax + + if (style := request.query.get("style")) is None: + style = self.config.default_style + + raw = request.query.get("raw") + + if raw is not None: + log_info(f"sending raw paste {key}") + + return web.HTTPOk(text=paste.text, content_type="text/plain") + else: + + def render(): + return pygmentize(paste.text, syntax, style, self.config.line_numbers) + + highlighted = await asyncio.to_thread(render) + + log_info( + f"sending rendered paste {key} with syntax {syntax} and style {style}" + ) + + return web.HTTPOk(text=highlighted, content_type="text/html") + + async def upload(self, request: web.Request) -> web.Response: + syntax = request.query.get("syntax") + + if ( + content_length := request.content_length + ) and content_length > self.config.content_max_bytes: + return web.HTTPBadRequest( + text=f"max content length is {self.config.content_max_bytes}" + ) + + try: + data = await request.read() + except Exception as e: + log_error(f"failed to read data: {e}") + return web.HTTPInternalServerError(text="failed to read data") + + try: + text = data.decode() + except UnicodeError: + return web.HTTPBadRequest( + text="content must be unicode only, no binary data is allowed" + ) + + key = generate_key(self.config.dictionary, self.config.key_length) + + try: + paste = Paste(key, datetime.now(), syntax, text) + await self.storage.insert(paste) + except Exception as e: + log_error(f"failed to insert paste {key} to storage: {e}") + return web.HTTPInternalServerError() + + url = f"{self.config.site}/paste/{key}" + + log_info( + f"uploaded paste {key} with syntax {syntax} of size {len(data)} bytes: {url}" + ) + + return web.HTTPOk(text=url) diff --git a/pypaste/server/__main__.py b/pypaste/server/__main__.py new file mode 100644 index 0000000..c28a9ba --- /dev/null +++ b/pypaste/server/__main__.py @@ -0,0 +1,163 @@ +# Copyright (C) 2025 John Turner + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import sys +import os +import asyncio +import aiosqlite +from pypaste import log_error, log_info +from pypaste.server import App, AppConfig, Storage +from pypaste.server.s3 import S3 +from pypaste.server.sqlite import Sqlite +from socket import socket, AF_UNIX, SOCK_STREAM +from argparse import ArgumentParser +from aiohttp import web +from pathlib import Path + + +async def main() -> int: + parser = ArgumentParser() + subparsers = parser.add_subparsers(dest="command") + + parser.add_argument("--socket", required=True) + parser.add_argument("--socket-mode", default="0600") + parser.add_argument("--site", required=True) + parser.add_argument("--content-length-max-bytes", type=int, required=True) + parser.add_argument("--key-length", type=int, required=True) + parser.add_argument("--dictionary", type=Path, required=True) + parser.add_argument("--database", type=Path, required=True) + parser.add_argument("--storage-max-bytes", type=int, required=True) + parser.add_argument("--default-style", default="native") + parser.add_argument("--line-numbers", action="store_true") + parser.add_argument("--line-numbers-inline", action="store_true") + + s3parser = subparsers.add_parser("s3") + s3parser.add_argument("--endpoint", required=True) + s3parser.add_argument("--region", required=True) + s3parser.add_argument("--bucket", required=True) + s3parser.add_argument("--access-key", required=True) + s3parser.add_argument("--secret-key", type=Path, required=True) + + subparsers.add_parser("sqlite") + + args = parser.parse_args() + + try: + dictionary = args.dictionary.read_text().split("\n") + except Exception as e: + print(f"failed to open dictionary: {str(args.dictionary)}: {e}") + return 1 + + match [args.line_numbers, args.line_numbers_inline]: + case [True, _]: + line_numbers: str | bool = "table" + case [_, True]: + line_numbers = "inline" + case [_, _]: + line_numbers = False + + config = AppConfig( + args.site, + args.content_length_max_bytes, + args.storage_max_bytes, + args.key_length, + dictionary, + args.default_style, + line_numbers, + ) + + try: + connection = await aiosqlite.connect(args.database) + except Exception as e: + log_error(f"failed to connect to database {args.database}: {e}") + return 1 + + try: + await connection.execute( + ( + "create table if not exists pastes(key text, datetime text, size int, syntax text)" + ) + ) + await connection.commit() + except Exception as e: + log_error(f"failed to initialize database: {e}") + return 1 + + match args.command: + case "s3": + try: + secret_key = args.secret_key.read_text().strip() + except Exception as e: + log_error(f"failed to load secret key from {args.secret_key}: {e}") + return 1 + + storage: Storage = S3( + connection, + args.endpoint, + args.region, + args.bucket, + args.access_key, + secret_key, + ) + + await storage.setup() + case "sqlite": + storage = Sqlite(connection) + + await storage.setup() + + case _: + raise Exception(f"unsupported storage backend: {args.command}") + + pypaste = App(config, storage) + + app = web.Application() + + app.add_routes( + [web.get("/paste/{key}", pypaste.download), web.post("/paste", pypaste.upload)] + ) + + runner = web.AppRunner(app, logger=None) + + await runner.setup() + + sock = socket(AF_UNIX, SOCK_STREAM) + sock.bind(args.socket) + + os.chmod(args.socket, int(args.socket_mode, 8)) + + site = web.SockSite(runner, sock) + + await site.start() + + try: + await asyncio.Event().wait() + except asyncio.exceptions.CancelledError: + pass + finally: + os.remove(args.socket) + + return 0 + + +if __name__ == "__main__": + try: + log_info("starting pypaste") + sys.exit(asyncio.run(main())) + except KeyboardInterrupt: + sys.exit(0) + except Exception as e: + log_error(str(e)) + sys.exit(1) diff --git a/pypaste/server/s3/__init__.py b/pypaste/server/s3/__init__.py new file mode 100644 index 0000000..07e2580 --- /dev/null +++ b/pypaste/server/s3/__init__.py @@ -0,0 +1,126 @@ +# Copyright (C) 2025 John Turner + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import asyncio +import zstandard +import asyncio +import aiosqlite +from pypaste.server import Storage, Paste +from pypaste.server.s3.bucket import Bucket +from dataclasses import dataclass +from typing import Optional + + +@dataclass +class S3(Storage): + connection: aiosqlite.Connection + + def __init__( + self, + connection: aiosqlite.Connection, + endpoint: str, + region: str, + bucket: str, + access_key: str, + secret_key: str, + ): + self.connection = connection + self.bucket = Bucket(endpoint, region, bucket, access_key, secret_key) + + async def setup(self) -> None: + await self.connection.execute("create table if not exists s3(key text)") + await self.connection.commit() + + async def insert(self, paste: Paste) -> None: + def compress(): + return zstandard.compress(paste.text.encode()) + + compressed = await asyncio.to_thread(compress) + + await self.connection.execute( + "insert into pastes values(?, ?, ?, ?)", + (paste.key, paste.dt.isoformat(), len(compressed), paste.syntax), + ) + + try: + await self.bucket.put(paste.key, compressed) + await self.connection.commit() + except Exception as e: + await self.connection.rollback() + raise e + + async def retrieve(self, key: str) -> Optional[Paste]: + if not await self.exists(key): + return None + + row = await self.read_row(key) + + assert row is not None + + (dt, size, syntax) = row + + data = await self.bucket.get(key) + + assert data is not None + + def decompress() -> str: + return zstandard.decompress(data).decode() + + text = await asyncio.to_thread(decompress) + + return Paste(key, dt, syntax, text) + + async def delete(self, key: str) -> None: + await self.connection.execute("delete from pastes where key=?", (key,)) + + try: + await self.bucket.delete(key) + await self.connection.commit() + except Exception as e: + await self.connection.rollback() + raise e + + async def vacuum(self, max: int) -> None: + while True: + async with self.connection.execute( + ( + "select sum(pastes.size) from pastes " + "inner join s3 on s3.key " + "where s3.key=pastes.key" + ) + ) as cursor: + if (row := await cursor.fetchone()) is None: + return + else: + use = row[0] + + async with self.connection.execute( + ( + "select pastes.key from pastes " + "inner join s3 on s3.key " + "where s3.key=pastes.key " + "order by pastes.datetime " + "limit 1" + ) + ) as cursor: + if (row := await cursor.fetchone()) is None: + return + else: + oldest = row[0] + + if use > max: + await self.delete(oldest) + else: + return diff --git a/pypaste/server/s3/bucket.py b/pypaste/server/s3/bucket.py new file mode 100644 index 0000000..c795bbd --- /dev/null +++ b/pypaste/server/s3/bucket.py @@ -0,0 +1,146 @@ +# Copyright (C) 2025 John Turner + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import aiohttp +from datetime import datetime, UTC +from typing import Dict +from dataclasses import dataclass +from bozo4 import s3v4_sign_request, s3v4_datetime_string +from hashlib import sha256 +from typing import Optional + + +@dataclass +class Request: + url: str + headers: Dict[str, str] + + +@dataclass +class Bucket: + endpoint: str + region: str + bucket: str + access_key: str + secret_key: str + + async def get(self, key: str) -> Optional[bytes]: + now = datetime.now(UTC) + + headers = { + "Host": self.endpoint, + "X-Amz-Date": s3v4_datetime_string(now), + "X-Amz-Content-SHA256": "UNSIGNED-PAYLOAD", + } + + auth = s3v4_sign_request( + endpoint=self.endpoint, + region=self.region, + access_key=self.access_key, + secret_key=self.secret_key, + request_method="GET", + date=now, + payload_hash="UNSIGNED-PAYLOAD", + uri=f"/{self.bucket}/{key}", + parameters={}, + headers=headers, + service="s3", + ) + + headers["Authorization"] = auth + + url = f"https://{self.endpoint}/{self.bucket}/{key}" + + async with aiohttp.ClientSession().get(url, headers=headers) as get: + match get.status: + case 200: + return await get.read() + case 404: + return None + case _: + raise Exception( + f"failed to get {self.endpoint}/{self.bucket}/{key} with status {get.status}" + ) + + async def put(self, key: str, data: bytes) -> None: + payload_hash = sha256(data).hexdigest() + + now = datetime.now(UTC) + + headers = { + "Host": self.endpoint, + "X-Amz-Date": s3v4_datetime_string(now), + "X-Amz-Content-SHA256": payload_hash, + "Content-Length": str(len(data)), + "Content-Type": "application/octect-stream", + } + + auth = s3v4_sign_request( + endpoint=self.endpoint, + region=self.region, + access_key=self.access_key, + secret_key=self.secret_key, + request_method="PUT", + date=now, + payload_hash=payload_hash, + uri=f"/{self.bucket}/{key}", + parameters={}, + headers=headers, + service="s3", + ) + + headers["Authorization"] = auth + + url = f"https://{self.endpoint}/{self.bucket}/{key}" + + async with aiohttp.ClientSession().put(url, headers=headers, data=data) as put: + if put.status != 200: + raise Exception( + f"failed put {self.endpoint}/{self.bucket}/{key} with {put.status}" + ) + + async def delete(self, key: str) -> None: + now = datetime.now(UTC) + + headers = { + "Host": self.endpoint, + "X-Amz-Date": s3v4_datetime_string(now), + "X-Amz-Content-SHA256": "UNSIGNED-PAYLOAD", + "Content-Length": "0", + } + + auth = s3v4_sign_request( + endpoint=self.endpoint, + region=self.region, + access_key=self.access_key, + secret_key=self.secret_key, + request_method="DELETE", + date=now, + payload_hash="UNSIGNED-PAYLOAD", + uri=f"/{self.bucket}/{key}", + parameters={}, + headers=headers, + service="s3", + ) + + headers["Authorization"] = auth + + url = f"https://{self.endpoint}/{self.bucket}/{key}" + + async with aiohttp.ClientSession().delete(url, headers=headers) as delete: + if delete.status != 200: + raise Exception( + f"failed to delete {self.endpoint}/{self.bucket}/{key} with {delete.status}" + ) diff --git a/pypaste/server/sqlite/__init__.py b/pypaste/server/sqlite/__init__.py new file mode 100644 index 0000000..2eb2ae5 --- /dev/null +++ b/pypaste/server/sqlite/__init__.py @@ -0,0 +1,103 @@ +import asyncio +import zstandard +import aiosqlite +from pypaste.server import Storage, Paste +from dataclasses import dataclass +from typing import Optional + + +@dataclass +class Sqlite(Storage): + connection: aiosqlite.Connection + + async def setup(self) -> None: + await self.connection.execute( + "create table if not exists sqlite(key text, data blob)" + ) + + await self.connection.commit() + + async def insert(self, paste: Paste) -> None: + def compress(): + return zstandard.compress(paste.text.encode()) + + data = await asyncio.to_thread(compress) + + await self.connection.execute( + "insert into pastes values(?, ?, ?, ?)", + (paste.key, paste.dt.isoformat(), len(data), paste.syntax), + ) + + await self.connection.execute( + "insert into sqlite values(?, ?)", + ( + paste.key, + data, + ), + ) + + await self.connection.commit() + + async def retrieve(self, key: str) -> Optional[Paste]: + if not await self.exists(key): + return None + + async with self.connection.execute( + "select sqlite.data from sqlite where key=? limit 1", (key,) + ) as cursor: + match await cursor.fetchone(): + case [bytes(data)]: + pass + case _: + raise Exception("unreachable") + + row = await self.read_row(key) + + assert row is not None + + (dt, size, syntax) = row + + def decompress(): + return zstandard.decompress(data).decode() + + text = await asyncio.to_thread(decompress) + + return Paste(key, dt, syntax, text) + + async def delete(self, key: str) -> None: + await self.connection.execute("delete from pastes where key=?", (key,)) + + await self.connection.execute("delete from sqlite where key=?", (key,)) + + async def vacuum(self, max: int) -> None: + while True: + async with self.connection.execute( + ( + "select sum(pastes.size) from pastes " + "inner join sqlite on sqlite.key " + "where pastes.key=sqlite.key" + ) + ) as cursor: + if (row := await cursor.fetchone()) is None: + return + else: + use = row[0] + + async with self.connection.execute( + ( + "select pastes.key from pastes " + "inner join sqlite on sqlite.key " + "where pastes.key=sqlite.key " + "order by pastes.datetime " + "limit 1" + ) + ) as cursor: + if (row := await cursor.fetchone()) is None: + return + else: + oldest = row[0] + + if use > max: + await self.delete(oldest) + else: + return diff --git a/pypaste/sqlite/__init__.py b/pypaste/sqlite/__init__.py deleted file mode 100644 index 5700356..0000000 --- a/pypaste/sqlite/__init__.py +++ /dev/null @@ -1,103 +0,0 @@ -import asyncio -import zstandard -import aiosqlite -from pypaste import Storage, Paste -from dataclasses import dataclass -from typing import Optional - - -@dataclass -class Sqlite(Storage): - connection: aiosqlite.Connection - - async def setup(self) -> None: - await self.connection.execute( - "create table if not exists sqlite(key text, data blob)" - ) - - await self.connection.commit() - - async def insert(self, paste: Paste) -> None: - def compress(): - return zstandard.compress(paste.text.encode()) - - data = await asyncio.to_thread(compress) - - await self.connection.execute( - "insert into pastes values(?, ?, ?, ?)", - (paste.key, paste.dt.isoformat(), len(data), paste.syntax), - ) - - await self.connection.execute( - "insert into sqlite values(?, ?)", - ( - paste.key, - data, - ), - ) - - await self.connection.commit() - - async def retrieve(self, key: str) -> Optional[Paste]: - if not await self.exists(key): - return None - - async with self.connection.execute( - "select sqlite.data from sqlite where key=? limit 1", (key,) - ) as cursor: - match await cursor.fetchone(): - case [bytes(data)]: - pass - case _: - raise Exception("unreachable") - - row = await self.read_row(key) - - assert row is not None - - (dt, size, syntax) = row - - def decompress(): - return zstandard.decompress(data).decode() - - text = await asyncio.to_thread(decompress) - - return Paste(key, dt, syntax, text) - - async def delete(self, key: str) -> None: - await self.connection.execute("delete from pastes where key=?", (key,)) - - await self.connection.execute("delete from sqlite where key=?", (key,)) - - async def vacuum(self, max: int) -> None: - while True: - async with self.connection.execute( - ( - "select sum(pastes.size) from pastes " - "inner join sqlite on sqlite.key " - "where pastes.key=sqlite.key" - ) - ) as cursor: - if (row := await cursor.fetchone()) is None: - return - else: - use = row[0] - - async with self.connection.execute( - ( - "select pastes.key from pastes " - "inner join sqlite on sqlite.key " - "where pastes.key=sqlite.key " - "order by pastes.datetime " - "limit 1" - ) - ) as cursor: - if (row := await cursor.fetchone()) is None: - return - else: - oldest = row[0] - - if use > max: - await self.delete(oldest) - else: - return diff --git a/pypaste/sqlite/meson.build b/pypaste/sqlite/meson.build deleted file mode 100644 index b1c4131..0000000 --- a/pypaste/sqlite/meson.build +++ /dev/null @@ -1 +0,0 @@ -sources += files('__init__.p'y) -- cgit v1.2.3