diff --git a/intbot/conftest.py b/intbot/conftest.py index 235f1de..d4e7459 100644 --- a/intbot/conftest.py +++ b/intbot/conftest.py @@ -3,6 +3,7 @@ from unittest import mock import pytest +from core.models import PretixData from django.conf import settings from django.db import connections @@ -22,6 +23,27 @@ def github_data(): } +@pytest.fixture(scope="session") +def pretix_data(): + """Pytest fixture with examples of data from pretix""" + base_path = settings.BASE_DIR / "tests" / "test_integrations" / "pretix" + + with open(base_path / "orders.json") as fd: + orders = json.load(fd) + + with open(base_path / "products.json") as fd: + products = json.load(fd) + + with open(base_path / "vouchers.json") as fd: + vouchers = json.load(fd) + + return { + "products": products, + "orders": orders, + "vouchers": vouchers, + } + + # NOTE(artcz) # The fixture below (fix_async_db) is copied from this issue # https://github.com/pytest-dev/pytest-asyncio/issues/226 diff --git a/intbot/core/analysis/orders.py b/intbot/core/analysis/orders.py new file mode 100644 index 0000000..0726db5 --- /dev/null +++ b/intbot/core/analysis/orders.py @@ -0,0 +1,186 @@ +""" +Parse Pretix Orders +""" + +from datetime import date +from decimal import Decimal + +import polars as pl +from core.analysis.products import ( + FlatProductDescription, + get_latest_products_data, + get_product_lookup, +) +from core.models import PretixData +from pydantic import BaseModel, ValidationInfo, model_validator + + +class InvoiceAddress(BaseModel): + company: str + country: str + vat_id: str + + +class OrderItem(BaseModel): + id: int + item: int + product: FlatProductDescription | None = None + attendee_email: str | None + attendee_name: str | None + price: Decimal + voucher: int | None = None + # answers: list[dict] + previous_conferences: str | None = None + addon_to: int | None + canceled: bool + order_date: date | None = None + country: str | None = None + company: str | None = None + affiliation: str | None = None + + class Questions: + previous_conferences = "S7CSFACW" + affiliation = "TWPG9CEQ" + + @model_validator(mode="before") + def extract(cls, values): + # Some things are available as answers to questions and we can extract + # them here + for answer in values["answers"]: + if cls.matches_question(answer, cls.Questions.previous_conferences): + values["previous_conferences"] = answer["answer"] + + if cls.matches_question(answer, cls.Questions.affiliation): + values["affiliation"] = answer["answer"] + + return values + + @model_validator(mode="before") + def map_product(cls, values, info: ValidationInfo): + # year = info.context["year"] + products = info.context["products_lookup"] + # passing product to make sure + values["product"] = products.get(values["item"], values["variation"]) + + return values + + @staticmethod + def matches_question(answer: dict, question: str) -> bool: + return answer.get("question_identifier", "") == question + + +class Order(BaseModel): + code: str + status: str + email: str + payment_date: date | None + total: Decimal + positions: list[OrderItem] + invoice_address: InvoiceAddress + status: str + + class Status: + PAID = "p" + + +class Ticket(BaseModel): + order_code: str + order_status: str + email: str | None + order_date: date | None + product_name: str + product_id: int + variation_id: int + product_type: str + full_name: str | None + country: str + company: str | None + affiliation: str | None + voucher: int | None + + +def get_latest_orders_data() -> PretixData: + return PretixData.objects.filter(resource=PretixData.PretixResources.orders).latest( + "created_at" + ) + + +def parse_latest_orders_to_objects( + *, + pretix_data_orders: PretixData, + pretix_data_products: PretixData, +) -> list[Order]: + products_lookup = get_product_lookup(pretix_data_products) + orders_data = pretix_data_orders.content + orders = [ + Order.model_validate(entry, context={"products_lookup": products_lookup}) + for entry in orders_data + ] + return orders + + +def flat_tickets(orders: list[Order]) -> list[Ticket]: + ONSITE_PRODUCTS = [ + "Business", + "Personal", + "Education", + "Presenter", + "Sponsor Conference Pass", + "Community Contributors", + "Grant ticket", + ] + + tickets = [] + for order in orders: + if order.status != Order.Status.PAID: + continue + + for pos in order.positions: + # NOTE: This should be a different check in the future, but for now + # it's good enough + + # Assume product already exists here + assert pos.product + + if pos.product.product_name in ONSITE_PRODUCTS: + ticket = Ticket( + order_code=order.code, + order_status=order.status, + order_date=order.payment_date, + product_name=pos.product.product_name, + product_id=pos.product.product_id, + variation_id=pos.product.variation_id, + product_type=pos.product.type, + email=pos.attendee_email, + full_name=pos.attendee_name, + affiliation=pos.affiliation, + country=order.invoice_address.country, + company=order.invoice_address.company, + voucher=pos.voucher, + ) + + tickets.append(ticket) + + + return tickets + +def flat_tickets_data(orders: list[Order]) -> pl.DataFrame: + tickets = flat_tickets(orders) + return pl.DataFrame(tickets, infer_schema_length=None) + + +def latest_flat_tickets_data() -> pl.DataFrame: + """ + Thin wrapper on getting latest information from the database, and + converting into a polars data frame + """ + orders_data = get_latest_orders_data() + products_data = get_latest_products_data() + orders = parse_latest_orders_to_objects( + pretix_data_orders=orders_data, pretix_data_products=products_data + ) + return flat_tickets_data(orders) + + +def group_tickets_by_product(tickets_data: pl.DataFrame) -> pl.DataFrame: + return tickets_data.group_by("product_name").len().sort("len", descending=True) diff --git a/intbot/core/analysis/pretix_shared.py b/intbot/core/analysis/pretix_shared.py new file mode 100644 index 0000000..e69de29 diff --git a/intbot/core/analysis/products.py b/intbot/core/analysis/products.py index e7acaa4..6a660d6 100644 --- a/intbot/core/analysis/products.py +++ b/intbot/core/analysis/products.py @@ -64,7 +64,7 @@ def parse_latest_products_to_objects(pretix_data: PretixData) -> list[Product]: return products -def flat_product_data(products: list[Product]) -> pl.DataFrame: +def flat_products(products: list[Product]) -> pl.DataFrame: """ Returns a polars data frame with flat description of available Products. Products hold nested `ProductVariation`s; flatten them so every variation @@ -106,7 +106,24 @@ def flat_product_data(products: list[Product]) -> pl.DataFrame: ) ) - return pl.DataFrame(rows) + return rows + + +def flat_product_data(products: list[Product]) -> pl.DataFrame: + return pl.DataFrame(flat_products(products)) + + +def get_product_lookup( + pretix_data: PretixData, +) -> dict[tuple[str, str], FlatProductDescription]: + """ + Returns a dictionary that maps product name/variant to a + FlatProductDescription instance. + Useful when parsing product name to a product instance + """ + products = flat_products(parse_latest_products_to_objects(pretix_data)) + + return {(product.product_id, product.variation_id): product for product in products} def latest_flat_product_data() -> pl.DataFrame: diff --git a/intbot/core/bot/main.py b/intbot/core/bot/main.py index 37df687..d44f9c4 100644 --- a/intbot/core/bot/main.py +++ b/intbot/core/bot/main.py @@ -2,6 +2,10 @@ import discord from asgiref.sync import sync_to_async +from core.analysis.orders import ( + group_tickets_by_product, + latest_flat_tickets_data, +) from core.analysis.products import latest_flat_product_data from core.analysis.submissions import ( group_submissions_by_state, @@ -247,6 +251,14 @@ async def submissions_status_pie_chart(ctx): await ctx.send(file=file) +@bot.command() +async def tickets_by_type(ctx): + df = await sync_to_async(latest_flat_tickets_data)() + by_product = group_tickets_by_product(df) + + await ctx.send(f"```{str(by_product)}```") + + def run_bot(): bot_token = settings.DISCORD_BOT_TOKEN bot.run(bot_token) diff --git a/intbot/tests/test_analysis/test_orders.py b/intbot/tests/test_analysis/test_orders.py new file mode 100644 index 0000000..aa55454 --- /dev/null +++ b/intbot/tests/test_analysis/test_orders.py @@ -0,0 +1,124 @@ +""" +Tickets tests +""" + +import polars as pl +import pytest +from core.analysis.orders import latest_flat_tickets_data +from core.models import PretixData +from polars.testing import assert_frame_equal + +random_order = { + "url": "https://tickets.europython.eu/order/CODE/secret-goes-here/", + "code": "CODE", + "fees": [], + "email": "artur+test@europython.eu", + "event": "ep2025", + "phone": None, + "total": "2.00", + "locale": "en", + "secret": "secret-goes-here", + "status": "p", + "comment": "", + "expires": "2025-04-10T23:59:59+02:00", + "refunds": [], + "api_meta": {}, + "customer": None, + "datetime": "2025-03-27T03:04:17+01:00", + "payments": [], + "testmode": False, + "positions": [ + { + "id": 1000, + "item": 100, + "answers": [ + { + "answer": "EPS", + "options": [], + "question": 184831, + "option_identifiers": [], + "question_identifier": "TWPG9CEQ", + }, + { + "answer": "35", + "options": [], + "question": 184832, + "option_identifiers": [], + "question_identifier": "3PGQBFXV", + }, + ], + "voucher": 8868743, + "variation": 1, + "positionid": 1, + "canceled": False, + "attendee_name": "Artur Czepiel", + "attendee_email": "artur+test@europython.eu", + }, + { + "id": 101, + "item": 725441, + "order": "CODE", + "price": "0.00", + "variation": None, + "canceled": False, + "secret": "t-shirt-secret", + "answers": [ + { + "answer": "L - straight cut", + "options": [335101], + "question": 184841, + "option_identifiers": ["WHUDNPJK"], + "question_identifier": "A8JP9TRN", + } + ], + }, + ], + "payment_date": "2025-03-27", + "last_modified": "2025-03-27T03:04:20+01:00", + "invoice_address": { + "city": "Krak\u00f3w", + "name": "Artur Czepiel", + "state": "", + "street": "Krakowska 123", + "vat_id": "", + "company": "", + "country": "PL", + "zipcode": "30-100", + "name_parts": { + "_scheme": "given_family", + "given_name": "Artur", + "family_name": "Czepiel", + }, + "is_business": False, + "custom_field": None, + "vat_id_validated": False, + "internal_reference": "No internal reference", + }, +} + + +@pytest.mark.django_db +def test_latest_flat_tickets_data(pretix_data): + """ + Big integrated tests doing a simple sanity check through all the layers. + Storage+Parsing+Reporting. + """ + PretixData.objects.create( + content=pretix_data["products"], + resource=PretixData.PretixResources.products, + ) + PretixData.objects.create( + content=pretix_data["orders"], + resource=PretixData.PretixResources.orders, + ) + + expected = pl.DataFrame( + { + "state": ["submitted", "withdrawn"], + "len": [2, 1], + } + ) + + df = latest_flat_tickets_data() + + assert_frame_equal(df, expected) diff --git a/intbot/tests/test_analysis/test_products.py b/intbot/tests/test_analysis/test_products.py index 95f0f2c..69fda52 100644 --- a/intbot/tests/test_analysis/test_products.py +++ b/intbot/tests/test_analysis/test_products.py @@ -13,6 +13,7 @@ from polars.testing import assert_frame_equal + def test_flat_product_data(): products = [ Product( @@ -53,138 +54,19 @@ def test_flat_product_data(): @pytest.mark.django_db -def test_latest_flat_product_data(): +def test_latest_flat_product_data(pretix_data): """ Bigger integrated tests going through everything from getting data from the database to returning a polars dataframe """ - # NOTE: Real data from pretix contains more details, this is abbreviated - # just for the necessary data we need for parsing. PretixData.objects.create( resource=PretixData.PretixResources.products, - content=[ - { - "id": 100, - "category": 2000, - "name": {"en": "Business"}, - "description": { - "en": "If your company pays for you to attend, or if you use Python professionally. When you purchase a Business Ticket, you help us keep the conference affordable for everyone. \r\nThank you!" - }, - "default_price": "500.00", - "admission": True, - "variations": [ - { - "id": 1, - "value": {"en": "Conference"}, - "active": True, - "description": { - "en": "Access to Conference Days & Sprint Weekend (16-20 July). Tutorials (14-15 July) are **NOT** included. To access Tutorial days please buy a Tutorial or Combined ticket.\r\n\r\n**Net price \u20ac500.00 + 21% Czech VAT**. \r\n\r\nAvailable until sold out or 27 June." - }, - "default_price": "605.00", - "price": "605.00", - }, - { - "id": 2, - "value": {"en": "Tutorials"}, - "active": True, - "description": { - "en": "Access to Workshop/Tutorial Days (14-15 July) and the Sprint Weekend (19-20 July), but **NOT** the main conference (16-18 July). \r\n**Net price \u20ac400.00+ 21% Czech VAT.**\r\n\r\nTutorial tickets are only available until 27 June" - }, - "default_price": "484.00", - "price": "484.00", - }, - { - "id": 3, - "value": {"en": "Combined (Conference + Tutorials)"}, - "active": True, - "description": { - "en": "Access to everything during the whole seven-day event (14-20 July).\r\n**Net price \u20ac800.00 + 21% Czech VAT.**\r\n\r\nAvailable until sold out or 27 June." - }, - "default_price": "968.00", - "price": "968.00", - }, - { - "id": 4, - "value": {"en": "Late Conference"}, - "active": True, - "description": { - "en": "Access to Conference Days & Sprint Weekend (16-20 July) & limited access to specific sponsored/special workshops during the Workshop/Tutorial Days (14-15 July).\r\n**Net price \u20ac750.00 + 21% Czech VAT**\r\n\r\nAvailable from 27 June or after regular Conference tickets are sold out." - }, - "default_price": "907.50", - "price": "907.50", - }, - { - "id": 5, - "value": {"en": "Late Combined"}, - "active": True, - "description": { - "en": "Access to everything during the whole seven-day event (14-20 July).\r\n**Net price \u20ac1,200.00 + 21% Czech VAT.**\r\n\r\nAvailable from 27 June or after regular Combined tickets are sold out." - }, - "default_price": "1452.00", - "price": "1452.00", - }, - ], - }, - { - "id": 200, - "category": 2000, - "name": {"en": "Personal"}, - "active": True, - "description": { - "en": "If you enjoy Python as a hobbyist or use it as a freelancer." - }, - "default_price": "300.00", - "variations": [ - { - "id": 6, - "value": {"en": "Conference"}, - "description": { - "en": "Access to Conference Days & Sprint Weekend (16-20 July). Tutorials (14-15 July) are **NOT** included. \r\nTo access Tutorial days please buy a Tutorial or Combined ticket.\r\nAvailable until sold out or 27 June." - }, - "default_price": "300.00", - "price": "300.00", - }, - { - "id": 7, - "value": {"en": "Tutorials"}, - "description": { - "en": "Access to Workshop/Tutorial Days (14-15 July) and the Sprint Weekend (19-20 July), but **NOT** the main conference (16-18 July).\r\n\r\nAvailable until sold out or 27 June." - }, - "default_price": "200.00", - "price": "200.00", - }, - { - "id": 8, - "value": {"en": "Combined (Conference + Tutorials)"}, - "description": { - "en": "Access to everything during the whole seven-day event (14-20 July).\r\n\r\nAvailable until sold out or 27 June." - }, - "position": 2, - "default_price": "450.00", - "price": "450.00", - }, - { - "id": 9, - "value": {"en": "Late Conference"}, - "description": { - "en": "Access to Conference Days & Sprint Weekend (16-20 July). Tutorials (14-15 July) are NOT included. To access Tutorial days please buy a Tutorial or Combined ticket.\r\n\r\nAvailable from 27 June or after regular Conference tickets are sold out." - }, - "default_price": "450.00", - "price": "450.00", - }, - { - "id": 10, - "value": {"en": "Late Combined"}, - "description": { - "en": "Access to everything during the whole seven-day event (14-20 July).\r\n\r\nAvailable from 27 June or after regular Combined tickets are sold out." - }, - "default_price": "675.00", - "price": "675.00", - }, - ], - }, - ], + content=pretix_data["products"], ) + + + # NOTE: Real data from pretix contains more details, this is abbreviated + # just for the necessary data we need for parsing. expected = pl.DataFrame( [ FlatProductDescription( diff --git a/intbot/tests/test_integrations/pretix/orders.json b/intbot/tests/test_integrations/pretix/orders.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/intbot/tests/test_integrations/pretix/orders.json @@ -0,0 +1 @@ +[] diff --git a/intbot/tests/test_integrations/pretix/products.json b/intbot/tests/test_integrations/pretix/products.json new file mode 100644 index 0000000..b7b4b2e --- /dev/null +++ b/intbot/tests/test_integrations/pretix/products.json @@ -0,0 +1,146 @@ +[ + { + "id": 100, + "category": 2000, + "name": { + "en": "Business" + }, + "description": { + "en": "If your company pays for you to attend, or if you use Python professionally. When you purchase a Business Ticket, you help us keep the conference affordable for everyone. \r\nThank you!" + }, + "default_price": "500.00", + "admission": true, + "variations": [ + { + "id": 1, + "value": { + "en": "Conference" + }, + "active": true, + "description": { + "en": "Access to Conference Days & Sprint Weekend (16-20 July). Tutorials (14-15 July) are **NOT** included. To access Tutorial days please buy a Tutorial or Combined ticket.\r\n\r\n**Net price \u20ac500.00 + 21% Czech VAT**. \r\n\r\nAvailable until sold out or 27 June." + }, + "default_price": "605.00", + "price": "605.00" + }, + { + "id": 2, + "value": { + "en": "Tutorials" + }, + "active": true, + "description": { + "en": "Access to Workshop/Tutorial Days (14-15 July) and the Sprint Weekend (19-20 July), but **NOT** the main conference (16-18 July). \r\n**Net price \u20ac400.00+ 21% Czech VAT.**\r\n\r\nTutorial tickets are only available until 27 June" + }, + "default_price": "484.00", + "price": "484.00" + }, + { + "id": 3, + "value": { + "en": "Combined (Conference + Tutorials)" + }, + "active": true, + "description": { + "en": "Access to everything during the whole seven-day event (14-20 July).\r\n**Net price \u20ac800.00 + 21% Czech VAT.**\r\n\r\nAvailable until sold out or 27 June." + }, + "default_price": "968.00", + "price": "968.00" + }, + { + "id": 4, + "value": { + "en": "Late Conference" + }, + "active": true, + "description": { + "en": "Access to Conference Days & Sprint Weekend (16-20 July) & limited access to specific sponsored/special workshops during the Workshop/Tutorial Days (14-15 July).\r\n**Net price \u20ac750.00 + 21% Czech VAT**\r\n\r\nAvailable from 27 June or after regular Conference tickets are sold out." + }, + "default_price": "907.50", + "price": "907.50" + }, + { + "id": 5, + "value": { + "en": "Late Combined" + }, + "active": true, + "description": { + "en": "Access to everything during the whole seven-day event (14-20 July).\r\n**Net price \u20ac1,200.00 + 21% Czech VAT.**\r\n\r\nAvailable from 27 June or after regular Combined tickets are sold out." + }, + "default_price": "1452.00", + "price": "1452.00" + } + ] + }, + { + "id": 200, + "category": 2000, + "name": { + "en": "Personal" + }, + "active": true, + "description": { + "en": "If you enjoy Python as a hobbyist or use it as a freelancer." + }, + "default_price": "300.00", + "variations": [ + { + "id": 6, + "value": { + "en": "Conference" + }, + "description": { + "en": "Access to Conference Days & Sprint Weekend (16-20 July). Tutorials (14-15 July) are **NOT** included. \r\nTo access Tutorial days please buy a Tutorial or Combined ticket.\r\nAvailable until sold out or 27 June." + }, + "default_price": "300.00", + "price": "300.00" + }, + { + "id": 7, + "value": { + "en": "Tutorials" + }, + "description": { + "en": "Access to Workshop/Tutorial Days (14-15 July) and the Sprint Weekend (19-20 July), but **NOT** the main conference (16-18 July).\r\n\r\nAvailable until sold out or 27 June." + }, + "default_price": "200.00", + "price": "200.00" + }, + { + "id": 8, + "value": { + "en": "Combined (Conference + Tutorials)" + }, + "description": { + "en": "Access to everything during the whole seven-day event (14-20 July).\r\n\r\nAvailable until sold out or 27 June." + }, + "position": 2, + "default_price": "450.00", + "price": "450.00" + }, + { + "id": 9, + "value": { + "en": "Late Conference" + }, + "description": { + "en": "Access to Conference Days & Sprint Weekend (16-20 July). Tutorials (14-15 July) are NOT included. To access Tutorial days please buy a Tutorial or Combined ticket.\r\n\r\nAvailable from 27 June or after regular Conference tickets are sold out." + }, + "default_price": "450.00", + "price": "450.00" + }, + { + "id": 10, + "value": { + "en": "Late Combined" + }, + "description": { + "en": "Access to everything during the whole seven-day event (14-20 July).\r\n\r\nAvailable from 27 June or after regular Combined tickets are sold out." + }, + "default_price": "675.00", + "price": "675.00" + } + ] + } +] diff --git a/intbot/tests/test_integrations/pretix/vouchers.json b/intbot/tests/test_integrations/pretix/vouchers.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/intbot/tests/test_integrations/pretix/vouchers.json @@ -0,0 +1 @@ +[]