paul/mailstats
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Set up playbooks with imapclient and matplotlib

Browse Source
This commit is contained in:
Paul Brinkmeier 2022-09-17 14:56:23 +02:00
commit 97719ec9bc
4 changed files with 724 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

364
Playground.ipynb Normal file
View File

@ -0,0 +1,364 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 2,
"id": "0bda9dff-b37e-4dd1-b842-24f8947937cf",
"metadata": {},
"outputs": [],
"source": [
"from imapclient import IMAPClient\n",
"import getpass"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "1a95dfc6-747a-4e9d-ba55-061c3e1e53f6",
"metadata": {},
"outputs": [],
"source": [
"host = \"imap.mailbox.org\"\n",
"port = 993"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "92854690-2cc0-45fa-a609-1a3dfbe0d2bf",
"metadata": {},
"outputs": [
{
"name": "stdin",
"output_type": "stream",
"text": [
" ········\n"
]
}
],
"source": [
"user = \"hallo@pbrinkmeier.de\"\n",
"pw = getpass.getpass()"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "dfb17802-c27e-4e82-af2a-b9ac5383071e",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"b'[CAPABILITY IMAP4rev1 SASL-IR LOGIN-REFERRALS ID ENABLE IDLE SORT SORT=DISPLAY THREAD=REFERENCES THREAD=REFS THREAD=ORDEREDSUBJECT MULTIAPPEND URL-PARTIAL CATENATE UNSELECT CHILDREN NAMESPACE UIDPLUS LIST-EXTENDED I18NLEVEL=1 CONDSTORE QRESYNC ESEARCH ESORT SEARCHRES WITHIN CONTEXT=SEARCH LIST-STATUS BINARY MOVE SNIPPET=FUZZY PREVIEW=FUZZY PREVIEW STATUS=SIZE SAVEDATE SEARCH=X-MIMEPART XDOVECOT ACL UIDPLUS LITERAL+ NOTIFY SPECIAL-USE FILTER=SIEVE COMPRESS=DEFLATE XAPPLEPUSHSERVICE QUOTA ACL RIGHTS=texk] Logged in'"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"server = IMAPClient(host, use_uid=True)\n",
"server.login(user, pw)"
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "2e1b4ab9-96bb-4cad-9420-2eedcb1c0372",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['Junk',\n",
" 'Trash',\n",
" 'Trash/Amazon 1',\n",
" 'Drafts',\n",
" 'Sent',\n",
" 'INBOX/Scalable',\n",
" 'INBOX/Channable',\n",
" 'INBOX/Channable/Kamernet',\n",
" 'INBOX/EasyCare',\n",
" 'INBOX/Sonstiges',\n",
" 'INBOX/TK',\n",
" 'INBOX/Spam@',\n",
" 'INBOX/Onlineshopping',\n",
" 'INBOX/Onlineshopping/Momox',\n",
" 'INBOX/Onlineshopping/E-Bay Kleinanzeigen',\n",
" 'INBOX/Onlineshopping/E-Shop Direct',\n",
" 'INBOX/Onlineshopping/Gamefound',\n",
" 'INBOX/Onlineshopping/Buffalo',\n",
" 'INBOX/Onlineshopping/Fantasywelt',\n",
" 'INBOX/Onlineshopping/Amazon',\n",
" 'INBOX/Privat',\n",
" 'INBOX/Uni',\n",
" 'INBOX/Uni/Tutorien',\n",
" 'INBOX/Uni/Tutorien/PP',\n",
" 'INBOX/Uni/Fachschaft',\n",
" 'INBOX/Dienste',\n",
" 'INBOX/Dienste/Contabo',\n",
" 'INBOX/Dienste/IONOS',\n",
" 'INBOX/Dienste/Steam',\n",
" 'INBOX/Dienste/paydirekt',\n",
" 'INBOX/Dienste/Bahn',\n",
" 'INBOX/Dienste/Strato',\n",
" 'INBOX/Dienste/Booking.com',\n",
" 'INBOX/Dienste/Pakete',\n",
" 'INBOX/Dienste/PayPal',\n",
" 'INBOX/Dienste/Mailbox',\n",
" 'INBOX']"
]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"[name for (meta, parent, name) in server.list_folders()]"
]
},
{
"cell_type": "code",
"execution_count": 20,
"id": "070194da-6cf0-4717-a55d-0e8c4338a897",
"metadata": {},
"outputs": [],
"source": [
"select_info = server.select_folder('INBOX/Uni/Fachschaft')"
]
},
{
"cell_type": "code",
"execution_count": 21,
"id": "671556ba-5f40-440e-98de-313a47e8274a",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{b'PERMANENTFLAGS': (b'\\\\Answered',\n",
" b'\\\\Flagged',\n",
" b'\\\\Deleted',\n",
" b'\\\\Seen',\n",
" b'\\\\Draft',\n",
" b'NonJunk',\n",
" b'$Forwarded',\n",
" b'$social',\n",
" b'$MailFlagBit1',\n",
" b'$NotJunk',\n",
" b'NotJunk',\n",
" b'$ITipAnalyzed',\n",
" b'\\\\*'),\n",
" b'CLOSED': [b''],\n",
" b'FLAGS': (b'\\\\Answered',\n",
" b'\\\\Flagged',\n",
" b'\\\\Deleted',\n",
" b'\\\\Seen',\n",
" b'\\\\Draft',\n",
" b'NonJunk',\n",
" b'$Forwarded',\n",
" b'$social',\n",
" b'$MailFlagBit1',\n",
" b'$NotJunk',\n",
" b'NotJunk',\n",
" b'$ITipAnalyzed'),\n",
" b'EXISTS': 7921,\n",
" b'RECENT': 0,\n",
" b'UIDVALIDITY': 1592410848,\n",
" b'UIDNEXT': 7929,\n",
" b'HIGHESTMODSEQ': 16372,\n",
" b'READ-WRITE': True}"
]
},
"execution_count": 21,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"select_info"
]
},
{
"cell_type": "code",
"execution_count": 34,
"id": "8fc437cd-f1d4-4e26-8f22-b24bc2f627ef",
"metadata": {},
"outputs": [],
"source": [
"message_ids = server.search()\n",
"messages = list(data for id, data in server.fetch(message_ids, ['ENVELOPE']).items())"
]
},
{
"cell_type": "code",
"execution_count": 55,
"id": "299ed420-ca62-4ac9-95f4-f0cd0afed205",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"7921"
]
},
"execution_count": 55,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(messages)"
]
},
{
"cell_type": "code",
"execution_count": 56,
"id": "b1a7a03b-fed0-40e1-be2e-2f4ed2d09fe5",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{b'SEQ': 1,\n",
" b'ENVELOPE': Envelope(date=datetime.datetime(2020, 6, 18, 8, 46, 27), subject=b'[fsmi-aktiv] Protokoll vom Fachschaftsrat am 2020-05-13', from_=(Address(name=b'FSMI-Homepage', route=None, mailbox=b'webmaster', host=b'fsmi.uni-karlsruhe.de'),), sender=(Address(name=b'Mathe-Info-aktiv', route=None, mailbox=b'mathe-info-aktiv-bounces', host=b'fsmi.uni-karlsruhe.de'),), reply_to=(Address(name=b'FSMI-Homepage', route=None, mailbox=b'webmaster', host=b'fsmi.uni-karlsruhe.de'),), to=(Address(name=None, route=None, mailbox=b'aktiv', host=b'fsmi.uni-karlsruhe.de'),), cc=None, bcc=None, in_reply_to=None, message_id=b'<1592462787.262119.17324.nullmailer@fsmi-www.fsmi.uni-karlsruhe.de>')}"
]
},
"execution_count": 56,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"messages[0]"
]
},
{
"cell_type": "code",
"execution_count": 57,
"id": "a93b166e-f8cf-4fe4-be1c-ba4a3a3d48e8",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"7827"
]
},
"execution_count": 57,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"tos = [list(message[b'ENVELOPE'].to) for message in messages if message[b'ENVELOPE'].to]\n",
"len(tos)"
]
},
{
"cell_type": "code",
"execution_count": 108,
"id": "68d34864-3b30-4d96-8681-1181e56e230d",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"11451"
]
},
"execution_count": 108,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"flat_tos = [f\"{to.mailbox.decode('utf-8')}@{to.host.decode('utf-8')}\" for tos_ in tos for to in tos_ if to.mailbox and to.host]\n",
"len(flat_tos)"
]
},
{
"cell_type": "code",
"execution_count": 109,
"id": "0ac4e032-b665-4c38-8615-f4168356f307",
"metadata": {},
"outputs": [],
"source": [
"from collections import Counter"
]
},
{
"cell_type": "code",
"execution_count": 124,
"id": "2c91b254-b9f5-4448-b5ea-a18a034e827e",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1278 aktiv@fsmi.uni-karlsruhe.de\n",
"1126 mathe-info@fsmi.uni-karlsruhe.de\n",
"1077 info@fsmi.uni-karlsruhe.de\n",
" 545 mathe@fsmi.uni-karlsruhe.de\n",
" 397 top@fsmi.uni-karlsruhe.de\n",
" 153 fsk@asta-kit.de\n",
" 131 webmaster@fsmi.uni-karlsruhe.de\n",
" 120 intern@fsmi.uni-karlsruhe.de\n",
" 118 paul.brinkmeier@fsmi.uni-karlsruhe.de\n",
" 113 offtopic@fsmi.uni-karlsruhe.de\n",
" 105 stupal@asta-kit.de\n",
" 92 info-klausuren@fsmi.uni-karlsruhe.de\n",
" 88 mitarbeiter@ira.uka.de\n",
" 81 fsr-einladung@fsmi.uni-karlsruhe.de\n",
" 65 fachschaft@physik.kit.edu\n",
" 65 Mitarbeiter_Mathe@mail.math.kit.edu\n",
" 60 fschembio@lists.kit.edu\n",
" 60 info@fs-etec.kit.edu\n",
" 60 info@fachschaft.org\n",
" 60 hallo@pbrinkmeier.de\n"
]
}
],
"source": [
"tos_count = list(Counter(flat_tos).items())\n",
"tos_count.sort(key=lambda x: -x[1])\n",
"\n",
"for to, count in tos_count[:20]:\n",
" print(f\"{count:4} {to}\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7e225426-5446-45a9-b7e9-11c15bedb54b",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python3 - Python",
"language": "python",
"name": "ipython_python"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

21
README.md Normal file
View File

@ -0,0 +1,21 @@
# mailstats
Generate graphs from an IMAP mailbox.
## Nix stuff
I use this repo to play around with Nix flakes.
### Run on M1
The Python OpenSSL package is currently broken on M1. To run Jupyter Lab using Rosetta emulation:
```
nix --system x86_64-darwin run .#jupyterlab
```
### Update a dependency
```
nix flake lock --update-input <input-name>
```

294
flake.lock generated Normal file
View File

@ -0,0 +1,294 @@
{
"nodes": {
"flake-compat": {
"flake": false,
"locked": {
"lastModified": 1631705391,
"narHash": "sha256-PzP6vikNJZiS7yANC6sZWQlIDf4E2MTckvAsJxwV0DQ=",
"owner": "teto",
"repo": "flake-compat",
"rev": "8e15c6e3c0f15d0687a2ab6ae92cc7fab896bfed",
"type": "github"
},
"original": {
"owner": "teto",
"ref": "support-packages",
"repo": "flake-compat",
"type": "github"
}
},
"flake-compat_2": {
"flake": false,
"locked": {
"lastModified": 1627913399,
"narHash": "sha256-hY8g6H2KFL8ownSiFeMOjwPC8P0ueXpCVEbxgda3pko=",
"owner": "edolstra",
"repo": "flake-compat",
"rev": "12c64ca55c1014cdc1b16ed5a804aa8576601ff2",
"type": "github"
},
"original": {
"owner": "edolstra",
"repo": "flake-compat",
"type": "github"
}
},
"flake-compat_3": {
"flake": false,
"locked": {
"lastModified": 1627913399,
"narHash": "sha256-hY8g6H2KFL8ownSiFeMOjwPC8P0ueXpCVEbxgda3pko=",
"owner": "edolstra",
"repo": "flake-compat",
"rev": "12c64ca55c1014cdc1b16ed5a804aa8576601ff2",
"type": "github"
},
"original": {
"owner": "edolstra",
"repo": "flake-compat",
"type": "github"
}
},
"flake-utils": {
"locked": {
"lastModified": 1659877975,
"narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"flake-utils_2": {
"locked": {
"lastModified": 1659877975,
"narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"flake-utils_3": {
"locked": {
"lastModified": 1631561581,
"narHash": "sha256-3VQMV5zvxaVLvqqUrNz3iJelLw30mIVSfZmAaauM3dA=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "7e5bf3925f6fbdfaf50a2a7ca0be2879c4261d19",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"flake-utils_4": {
"locked": {
"lastModified": 1629481132,
"narHash": "sha256-JHgasjPR0/J1J3DRm4KxM4zTyAj4IOJY8vIl75v/kPI=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "997f7efcb746a9c140ce1f13c72263189225f482",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"gitignore": {
"flake": false,
"locked": {
"lastModified": 1611672876,
"narHash": "sha256-qHu3uZ/o9jBHiA3MEKHJ06k7w4heOhA+4HCSIvflRxo=",
"owner": "hercules-ci",
"repo": "gitignore.nix",
"rev": "211907489e9f198594c0eb0ca9256a1949c9d412",
"type": "github"
},
"original": {
"owner": "hercules-ci",
"repo": "gitignore.nix",
"type": "github"
}
},
"hls": {
"inputs": {
"flake-compat": "flake-compat_3",
"flake-utils": "flake-utils_4",
"gitignore": "gitignore",
"nixpkgs": "nixpkgs",
"pre-commit-hooks": "pre-commit-hooks"
},
"locked": {
"lastModified": 1637213318,
"narHash": "sha256-ZgxPwV7t4DyGYP7aXoetq+JHtd73XlOV2fYSflQmOXw=",
"owner": "haskell",
"repo": "haskell-language-server",
"rev": "311107eabbf0537e0c192b2c377d282505b4eff1",
"type": "github"
},
"original": {
"owner": "haskell",
"repo": "haskell-language-server",
"type": "github"
}
},
"ihaskell": {
"inputs": {
"flake-compat": "flake-compat_2",
"flake-utils": "flake-utils_3",
"hls": "hls",
"nixpkgs": "nixpkgs_2"
},
"locked": {
"lastModified": 1661486016,
"narHash": "sha256-Ch9dQ82fLV0QuyrJ+J1tVxxrSLSA+NIdiikvV0+Hvrs=",
"owner": "gibiansky",
"repo": "IHaskell",
"rev": "53f8e0773822ddf2cd392309ba27c8ff8e30202e",
"type": "github"
},
"original": {
"owner": "gibiansky",
"repo": "IHaskell",
"type": "github"
}
},
"jupyterWith": {
"inputs": {
"flake-compat": "flake-compat",
"flake-utils": "flake-utils_2",
"ihaskell": "ihaskell",
"nixpkgs": "nixpkgs_3"
},
"locked": {
"lastModified": 1663275532,
"narHash": "sha256-9wqqXRPDCEb/U4Wwmc6K9B28IOxZZXVmOwcy4vOiYL0=",
"owner": "tweag",
"repo": "jupyterwith",
"rev": "deaa6c66165fd1ebe8617a8f133ad45110ac659c",
"type": "github"
},
"original": {
"owner": "tweag",
"repo": "jupyterwith",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1630887066,
"narHash": "sha256-0ecIlrLsNIIa+zrNmzXXmbMBLZlmHU/aWFsa4bq99Hk=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "5e47a07e9f2d7ed999f2c7943b0896f5f7321ca3",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixpkgs-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs_2": {
"locked": {
"lastModified": 1634515797,
"narHash": "sha256-elgCUC2khtBkOSpE4gDymNvthTZAI4hGI2iNu3YEUkA=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "5f0194220f2402b06f7f79bba6351895facb5acb",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs_3": {
"locked": {
"lastModified": 1659446231,
"narHash": "sha256-hekabNdTdgR/iLsgce5TGWmfIDZ86qjPhxDg/8TlzhE=",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "eabc38219184cc3e04a974fe31857d8e0eac098d",
"type": "github"
},
"original": {
"owner": "nixos",
"ref": "release-21.11",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs_4": {
"locked": {
"lastModified": 1663345214,
"narHash": "sha256-dX6Ks6OFD59xq/LQUREHZM886AMhALUjDr9mLFAcKew=",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "b515e72bd7e9613021d62f7604c43e3d9efaf051",
"type": "github"
},
"original": {
"owner": "nixos",
"repo": "nixpkgs",
"type": "github"
}
},
"pre-commit-hooks": {
"inputs": {
"flake-utils": [
"jupyterWith",
"ihaskell",
"hls",
"flake-utils"
],
"nixpkgs": [
"jupyterWith",
"ihaskell",
"hls",
"nixpkgs"
]
},
"locked": {
"lastModified": 1624971177,
"narHash": "sha256-Amf/nBj1E77RmbSSmV+hg6YOpR+rddCbbVgo5C7BS0I=",
"owner": "cachix",
"repo": "pre-commit-hooks.nix",
"rev": "397f0713d007250a2c7a745e555fa16c5dc8cadb",
"type": "github"
},
"original": {
"owner": "cachix",
"repo": "pre-commit-hooks.nix",
"type": "github"
}
},
"root": {
"inputs": {
"flake-utils": "flake-utils",
"jupyterWith": "jupyterWith",
"nixpkgs": "nixpkgs_4"
}
}
},
"root": "root",
"version": 7
}

45
flake.nix Normal file
View File

@ -0,0 +1,45 @@
{
description = "Jupyter server with dependencies for mailstats";
inputs = {
# lib
nixpkgs.url = github:nixos/nixpkgs;
flake-utils.url = github:numtide/flake-utils;
# deps
jupyterWith.url = github:tweag/jupyterwith;
};
outputs = { self, nixpkgs, flake-utils, jupyterWith }: flake-utils.lib.eachDefaultSystem (system:
let
pkgs = import nixpkgs {
inherit system;
overlays = nixpkgs.lib.attrValues jupyterWith.overlays;
};
jupyterEnv = pkgs.jupyterlabWith {
kernels = [
ipython
];
};
ipython = pkgs.kernels.iPythonWith {
name = "Python";
packages = p: with p; [
imapclient
numpy
matplotlib
];
ignoreCollisions = true;
};
in
rec {
apps.default = apps.jupyterlab;
apps.jupyterlab = {
type = "app";
program = "${jupyterEnv}/bin/jupyter-lab";
};
}
);
}