From 97719ec9bc9d0a5d218e4e45ac8c9889afcc1bc6 Mon Sep 17 00:00:00 2001 From: Paul Brinkmeier Date: Sat, 17 Sep 2022 14:56:23 +0200 Subject: [PATCH] Set up playbooks with imapclient and matplotlib --- Playground.ipynb | 364 +++++++++++++++++++++++++++++++++++++++++++++++ README.md | 21 +++ flake.lock | 294 ++++++++++++++++++++++++++++++++++++++ flake.nix | 45 ++++++ 4 files changed, 724 insertions(+) create mode 100644 Playground.ipynb create mode 100644 README.md create mode 100644 flake.lock create mode 100644 flake.nix diff --git a/Playground.ipynb b/Playground.ipynb new file mode 100644 index 0000000..ca8ccde --- /dev/null +++ b/Playground.ipynb @@ -0,0 +1,364 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "id": "0bda9dff-b37e-4dd1-b842-24f8947937cf", + "metadata": {}, + "outputs": [], + "source": [ + "from imapclient import IMAPClient\n", + "import getpass" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "1a95dfc6-747a-4e9d-ba55-061c3e1e53f6", + "metadata": {}, + "outputs": [], + "source": [ + "host = \"imap.mailbox.org\"\n", + "port = 993" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "92854690-2cc0-45fa-a609-1a3dfbe0d2bf", + "metadata": {}, + "outputs": [ + { + "name": "stdin", + "output_type": "stream", + "text": [ + " ········\n" + ] + } + ], + "source": [ + "user = \"hallo@pbrinkmeier.de\"\n", + "pw = getpass.getpass()" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "dfb17802-c27e-4e82-af2a-b9ac5383071e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "b'[CAPABILITY IMAP4rev1 SASL-IR LOGIN-REFERRALS ID ENABLE IDLE SORT SORT=DISPLAY THREAD=REFERENCES THREAD=REFS THREAD=ORDEREDSUBJECT MULTIAPPEND URL-PARTIAL CATENATE UNSELECT CHILDREN NAMESPACE UIDPLUS LIST-EXTENDED I18NLEVEL=1 CONDSTORE QRESYNC ESEARCH ESORT SEARCHRES WITHIN CONTEXT=SEARCH LIST-STATUS BINARY MOVE SNIPPET=FUZZY PREVIEW=FUZZY PREVIEW STATUS=SIZE SAVEDATE SEARCH=X-MIMEPART XDOVECOT ACL UIDPLUS LITERAL+ NOTIFY SPECIAL-USE FILTER=SIEVE COMPRESS=DEFLATE XAPPLEPUSHSERVICE QUOTA ACL RIGHTS=texk] Logged in'" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "server = IMAPClient(host, use_uid=True)\n", + "server.login(user, pw)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "2e1b4ab9-96bb-4cad-9420-2eedcb1c0372", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['Junk',\n", + " 'Trash',\n", + " 'Trash/Amazon 1',\n", + " 'Drafts',\n", + " 'Sent',\n", + " 'INBOX/Scalable',\n", + " 'INBOX/Channable',\n", + " 'INBOX/Channable/Kamernet',\n", + " 'INBOX/EasyCare',\n", + " 'INBOX/Sonstiges',\n", + " 'INBOX/TK',\n", + " 'INBOX/Spam@',\n", + " 'INBOX/Onlineshopping',\n", + " 'INBOX/Onlineshopping/Momox',\n", + " 'INBOX/Onlineshopping/E-Bay Kleinanzeigen',\n", + " 'INBOX/Onlineshopping/E-Shop Direct',\n", + " 'INBOX/Onlineshopping/Gamefound',\n", + " 'INBOX/Onlineshopping/Buffalo',\n", + " 'INBOX/Onlineshopping/Fantasywelt',\n", + " 'INBOX/Onlineshopping/Amazon',\n", + " 'INBOX/Privat',\n", + " 'INBOX/Uni',\n", + " 'INBOX/Uni/Tutorien',\n", + " 'INBOX/Uni/Tutorien/PP',\n", + " 'INBOX/Uni/Fachschaft',\n", + " 'INBOX/Dienste',\n", + " 'INBOX/Dienste/Contabo',\n", + " 'INBOX/Dienste/IONOS',\n", + " 'INBOX/Dienste/Steam',\n", + " 'INBOX/Dienste/paydirekt',\n", + " 'INBOX/Dienste/Bahn',\n", + " 'INBOX/Dienste/Strato',\n", + " 'INBOX/Dienste/Booking.com',\n", + " 'INBOX/Dienste/Pakete',\n", + " 'INBOX/Dienste/PayPal',\n", + " 'INBOX/Dienste/Mailbox',\n", + " 'INBOX']" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "[name for (meta, parent, name) in server.list_folders()]" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "070194da-6cf0-4717-a55d-0e8c4338a897", + "metadata": {}, + "outputs": [], + "source": [ + "select_info = server.select_folder('INBOX/Uni/Fachschaft')" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "671556ba-5f40-440e-98de-313a47e8274a", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{b'PERMANENTFLAGS': (b'\\\\Answered',\n", + " b'\\\\Flagged',\n", + " b'\\\\Deleted',\n", + " b'\\\\Seen',\n", + " b'\\\\Draft',\n", + " b'NonJunk',\n", + " b'$Forwarded',\n", + " b'$social',\n", + " b'$MailFlagBit1',\n", + " b'$NotJunk',\n", + " b'NotJunk',\n", + " b'$ITipAnalyzed',\n", + " b'\\\\*'),\n", + " b'CLOSED': [b''],\n", + " b'FLAGS': (b'\\\\Answered',\n", + " b'\\\\Flagged',\n", + " b'\\\\Deleted',\n", + " b'\\\\Seen',\n", + " b'\\\\Draft',\n", + " b'NonJunk',\n", + " b'$Forwarded',\n", + " b'$social',\n", + " b'$MailFlagBit1',\n", + " b'$NotJunk',\n", + " b'NotJunk',\n", + " b'$ITipAnalyzed'),\n", + " b'EXISTS': 7921,\n", + " b'RECENT': 0,\n", + " b'UIDVALIDITY': 1592410848,\n", + " b'UIDNEXT': 7929,\n", + " b'HIGHESTMODSEQ': 16372,\n", + " b'READ-WRITE': True}" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "select_info" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "8fc437cd-f1d4-4e26-8f22-b24bc2f627ef", + "metadata": {}, + "outputs": [], + "source": [ + "message_ids = server.search()\n", + "messages = list(data for id, data in server.fetch(message_ids, ['ENVELOPE']).items())" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "id": "299ed420-ca62-4ac9-95f4-f0cd0afed205", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "7921" + ] + }, + "execution_count": 55, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(messages)" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "id": "b1a7a03b-fed0-40e1-be2e-2f4ed2d09fe5", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{b'SEQ': 1,\n", + " b'ENVELOPE': Envelope(date=datetime.datetime(2020, 6, 18, 8, 46, 27), subject=b'[fsmi-aktiv] Protokoll vom Fachschaftsrat am 2020-05-13', from_=(Address(name=b'FSMI-Homepage', route=None, mailbox=b'webmaster', host=b'fsmi.uni-karlsruhe.de'),), sender=(Address(name=b'Mathe-Info-aktiv', route=None, mailbox=b'mathe-info-aktiv-bounces', host=b'fsmi.uni-karlsruhe.de'),), reply_to=(Address(name=b'FSMI-Homepage', route=None, mailbox=b'webmaster', host=b'fsmi.uni-karlsruhe.de'),), to=(Address(name=None, route=None, mailbox=b'aktiv', host=b'fsmi.uni-karlsruhe.de'),), cc=None, bcc=None, in_reply_to=None, message_id=b'<1592462787.262119.17324.nullmailer@fsmi-www.fsmi.uni-karlsruhe.de>')}" + ] + }, + "execution_count": 56, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "messages[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 57, + "id": "a93b166e-f8cf-4fe4-be1c-ba4a3a3d48e8", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "7827" + ] + }, + "execution_count": 57, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tos = [list(message[b'ENVELOPE'].to) for message in messages if message[b'ENVELOPE'].to]\n", + "len(tos)" + ] + }, + { + "cell_type": "code", + "execution_count": 108, + "id": "68d34864-3b30-4d96-8681-1181e56e230d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "11451" + ] + }, + "execution_count": 108, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "flat_tos = [f\"{to.mailbox.decode('utf-8')}@{to.host.decode('utf-8')}\" for tos_ in tos for to in tos_ if to.mailbox and to.host]\n", + "len(flat_tos)" + ] + }, + { + "cell_type": "code", + "execution_count": 109, + "id": "0ac4e032-b665-4c38-8615-f4168356f307", + "metadata": {}, + "outputs": [], + "source": [ + "from collections import Counter" + ] + }, + { + "cell_type": "code", + "execution_count": 124, + "id": "2c91b254-b9f5-4448-b5ea-a18a034e827e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1278 aktiv@fsmi.uni-karlsruhe.de\n", + "1126 mathe-info@fsmi.uni-karlsruhe.de\n", + "1077 info@fsmi.uni-karlsruhe.de\n", + " 545 mathe@fsmi.uni-karlsruhe.de\n", + " 397 top@fsmi.uni-karlsruhe.de\n", + " 153 fsk@asta-kit.de\n", + " 131 webmaster@fsmi.uni-karlsruhe.de\n", + " 120 intern@fsmi.uni-karlsruhe.de\n", + " 118 paul.brinkmeier@fsmi.uni-karlsruhe.de\n", + " 113 offtopic@fsmi.uni-karlsruhe.de\n", + " 105 stupal@asta-kit.de\n", + " 92 info-klausuren@fsmi.uni-karlsruhe.de\n", + " 88 mitarbeiter@ira.uka.de\n", + " 81 fsr-einladung@fsmi.uni-karlsruhe.de\n", + " 65 fachschaft@physik.kit.edu\n", + " 65 Mitarbeiter_Mathe@mail.math.kit.edu\n", + " 60 fschembio@lists.kit.edu\n", + " 60 info@fs-etec.kit.edu\n", + " 60 info@fachschaft.org\n", + " 60 hallo@pbrinkmeier.de\n" + ] + } + ], + "source": [ + "tos_count = list(Counter(flat_tos).items())\n", + "tos_count.sort(key=lambda x: -x[1])\n", + "\n", + "for to, count in tos_count[:20]:\n", + " print(f\"{count:4} {to}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7e225426-5446-45a9-b7e9-11c15bedb54b", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python3 - Python", + "language": "python", + "name": "ipython_python" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/README.md b/README.md new file mode 100644 index 0000000..67fd258 --- /dev/null +++ b/README.md @@ -0,0 +1,21 @@ +# mailstats + +Generate graphs from an IMAP mailbox. + +## Nix stuff + +I use this repo to play around with Nix flakes. + +### Run on M1 + +The Python OpenSSL package is currently broken on M1. To run Jupyter Lab using Rosetta emulation: + +``` +nix --system x86_64-darwin run .#jupyterlab +``` + +### Update a dependency + +``` +nix flake lock --update-input +``` diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..a05618f --- /dev/null +++ b/flake.lock @@ -0,0 +1,294 @@ +{ + "nodes": { + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1631705391, + "narHash": "sha256-PzP6vikNJZiS7yANC6sZWQlIDf4E2MTckvAsJxwV0DQ=", + "owner": "teto", + "repo": "flake-compat", + "rev": "8e15c6e3c0f15d0687a2ab6ae92cc7fab896bfed", + "type": "github" + }, + "original": { + "owner": "teto", + "ref": "support-packages", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-compat_2": { + "flake": false, + "locked": { + "lastModified": 1627913399, + "narHash": "sha256-hY8g6H2KFL8ownSiFeMOjwPC8P0ueXpCVEbxgda3pko=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "12c64ca55c1014cdc1b16ed5a804aa8576601ff2", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-compat_3": { + "flake": false, + "locked": { + "lastModified": 1627913399, + "narHash": "sha256-hY8g6H2KFL8ownSiFeMOjwPC8P0ueXpCVEbxgda3pko=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "12c64ca55c1014cdc1b16ed5a804aa8576601ff2", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-utils": { + "locked": { + "lastModified": 1659877975, + "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_2": { + "locked": { + "lastModified": 1659877975, + "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_3": { + "locked": { + "lastModified": 1631561581, + "narHash": "sha256-3VQMV5zvxaVLvqqUrNz3iJelLw30mIVSfZmAaauM3dA=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "7e5bf3925f6fbdfaf50a2a7ca0be2879c4261d19", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_4": { + "locked": { + "lastModified": 1629481132, + "narHash": "sha256-JHgasjPR0/J1J3DRm4KxM4zTyAj4IOJY8vIl75v/kPI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "997f7efcb746a9c140ce1f13c72263189225f482", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "gitignore": { + "flake": false, + "locked": { + "lastModified": 1611672876, + "narHash": "sha256-qHu3uZ/o9jBHiA3MEKHJ06k7w4heOhA+4HCSIvflRxo=", + "owner": "hercules-ci", + "repo": "gitignore.nix", + "rev": "211907489e9f198594c0eb0ca9256a1949c9d412", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "gitignore.nix", + "type": "github" + } + }, + "hls": { + "inputs": { + "flake-compat": "flake-compat_3", + "flake-utils": "flake-utils_4", + "gitignore": "gitignore", + "nixpkgs": "nixpkgs", + "pre-commit-hooks": "pre-commit-hooks" + }, + "locked": { + "lastModified": 1637213318, + "narHash": "sha256-ZgxPwV7t4DyGYP7aXoetq+JHtd73XlOV2fYSflQmOXw=", + "owner": "haskell", + "repo": "haskell-language-server", + "rev": "311107eabbf0537e0c192b2c377d282505b4eff1", + "type": "github" + }, + "original": { + "owner": "haskell", + "repo": "haskell-language-server", + "type": "github" + } + }, + "ihaskell": { + "inputs": { + "flake-compat": "flake-compat_2", + "flake-utils": "flake-utils_3", + "hls": "hls", + "nixpkgs": "nixpkgs_2" + }, + "locked": { + "lastModified": 1661486016, + "narHash": "sha256-Ch9dQ82fLV0QuyrJ+J1tVxxrSLSA+NIdiikvV0+Hvrs=", + "owner": "gibiansky", + "repo": "IHaskell", + "rev": "53f8e0773822ddf2cd392309ba27c8ff8e30202e", + "type": "github" + }, + "original": { + "owner": "gibiansky", + "repo": "IHaskell", + "type": "github" + } + }, + "jupyterWith": { + "inputs": { + "flake-compat": "flake-compat", + "flake-utils": "flake-utils_2", + "ihaskell": "ihaskell", + "nixpkgs": "nixpkgs_3" + }, + "locked": { + "lastModified": 1663275532, + "narHash": "sha256-9wqqXRPDCEb/U4Wwmc6K9B28IOxZZXVmOwcy4vOiYL0=", + "owner": "tweag", + "repo": "jupyterwith", + "rev": "deaa6c66165fd1ebe8617a8f133ad45110ac659c", + "type": "github" + }, + "original": { + "owner": "tweag", + "repo": "jupyterwith", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1630887066, + "narHash": "sha256-0ecIlrLsNIIa+zrNmzXXmbMBLZlmHU/aWFsa4bq99Hk=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "5e47a07e9f2d7ed999f2c7943b0896f5f7321ca3", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1634515797, + "narHash": "sha256-elgCUC2khtBkOSpE4gDymNvthTZAI4hGI2iNu3YEUkA=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "5f0194220f2402b06f7f79bba6351895facb5acb", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_3": { + "locked": { + "lastModified": 1659446231, + "narHash": "sha256-hekabNdTdgR/iLsgce5TGWmfIDZ86qjPhxDg/8TlzhE=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "eabc38219184cc3e04a974fe31857d8e0eac098d", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "release-21.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_4": { + "locked": { + "lastModified": 1663345214, + "narHash": "sha256-dX6Ks6OFD59xq/LQUREHZM886AMhALUjDr9mLFAcKew=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "b515e72bd7e9613021d62f7604c43e3d9efaf051", + "type": "github" + }, + "original": { + "owner": "nixos", + "repo": "nixpkgs", + "type": "github" + } + }, + "pre-commit-hooks": { + "inputs": { + "flake-utils": [ + "jupyterWith", + "ihaskell", + "hls", + "flake-utils" + ], + "nixpkgs": [ + "jupyterWith", + "ihaskell", + "hls", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1624971177, + "narHash": "sha256-Amf/nBj1E77RmbSSmV+hg6YOpR+rddCbbVgo5C7BS0I=", + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "rev": "397f0713d007250a2c7a745e555fa16c5dc8cadb", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "jupyterWith": "jupyterWith", + "nixpkgs": "nixpkgs_4" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..48c044b --- /dev/null +++ b/flake.nix @@ -0,0 +1,45 @@ +{ + description = "Jupyter server with dependencies for mailstats"; + + inputs = { + # lib + nixpkgs.url = github:nixos/nixpkgs; + flake-utils.url = github:numtide/flake-utils; + + # deps + jupyterWith.url = github:tweag/jupyterwith; + }; + + outputs = { self, nixpkgs, flake-utils, jupyterWith }: flake-utils.lib.eachDefaultSystem (system: + let + pkgs = import nixpkgs { + inherit system; + overlays = nixpkgs.lib.attrValues jupyterWith.overlays; + }; + + jupyterEnv = pkgs.jupyterlabWith { + kernels = [ + ipython + ]; + }; + + ipython = pkgs.kernels.iPythonWith { + name = "Python"; + packages = p: with p; [ + imapclient + numpy + matplotlib + ]; + ignoreCollisions = true; + }; + in + rec { + apps.default = apps.jupyterlab; + + apps.jupyterlab = { + type = "app"; + program = "${jupyterEnv}/bin/jupyter-lab"; + }; + } + ); +}