Commit 2918a9b7 by Nicolas Joyard

Ajout script de parsing emails cada

parent e9175c58
......@@ -4,7 +4,8 @@
* Python 3 + headers (ie. python3-dev)
* virtualenvwrapper
* PostgreSQL (postgresql-server-dev-all)
* PostgreSQL (postgresql-server-dev-all) avec l'extension unaccent
* pdftotext
### Nouvelle Installation
......@@ -15,6 +16,7 @@ $ mkvirtualenv --python=$(which python3) irfm
$ pip install -e .
$ sudo -u postgres psql -c "create user irfm with password 'irfm';"
$ sudo -u postgres psql -c "create database irfm with owner irfm;"
$ sudo -u postgres psql irfm -c "create extension unaccent;"
$ irfm db upgrade
$ irfm import_nd
$ irfm import_adresses
......
......@@ -19,7 +19,8 @@ from .tools.files import generer_demandes as generer_demandes_
from .tools.mails import (envoyer_emails as envoyer_emails_,
envoyer_relances as envoyer_relances_,
mailing_lists as mailing_lists_,
erratum_cada as erratum_cada_)
erratum_cada as erratum_cada_,
extraire_mails_cada as extraire_mails_cada_)
from .tools.procedure import (fix_procedure as fix_procedure_,
avance_procedure as avance_procedure_)
from .tools.text import hash_password
......@@ -107,6 +108,15 @@ def erratum_cada():
@manager.command
def extraire_mails_cada():
"""
Extraire les e-mails de réponse CADA
"""
app.config.update(SQLALCHEMY_ECHO=False)
extraire_mails_cada_(app)
@manager.command
def generer_demandes():
"""Génère les demandes pour tous les parlementaires"""
app.config.update(SQLALCHEMY_ECHO=False)
......
# -*- coding: utf-8 -*-
from sqlalchemy.sql.functions import ReturnTypeFromArgs
class unaccent(ReturnTypeFromArgs):
pass
class replace(ReturnTypeFromArgs):
pass
def normalize_name(field):
return unaccent(replace(field, '-', ' '))
# -*- coding: utf-8 -*-
from datetime import datetime, timedelta
import email
import os
import re
import subprocess
import time
from flask import render_template
......@@ -9,10 +12,12 @@ from flask import render_template
from flask_mail import Mail, Message
from sqlalchemy.orm import contains_eager, joinedload
from sqlalchemy.orm.exc import NoResultFound
from ..models import Action, User, Parlementaire, db
from ..models.constants import (DELAI_RELANCE, DELAI_REPONSE, ETAPE_NA,
ETAPE_A_CONFIRMER, ETAPE_DEMANDE_CADA)
from ..models.functions import normalize_name
from ..tools.files import generer_demande
from ..tools.text import create_usertoken as token
......@@ -210,3 +215,60 @@ def erratum_cada(app):
print('%s e-mails d\'erratum envoyés' % len(messages))
time.sleep(1)
AVIS_INCOMPETENCE = 'La commission ne peut donc que se déclarer ' + \
'incompétente pour se prononcer sur la demande.'
AVIS_RE_DEPUTE = re.compile('par (Monsieur|Madame) ([^,]*),? députée?')
def extraire_mails_cada(app):
emails_root = os.path.join(app.config['DATA_DIR'], 'emails')
files_root = os.path.join(app.config['DATA_DIR'], 'files')
for eml in [f for f in os.listdir(emails_root)
if os.path.isfile(os.path.join(emails_root, f))
and f.endswith('.eml')]:
with open(os.path.join(emails_root, eml)) as f:
message = email.message_from_file(f)
cada_id = message['subject']
# Extraction pièce jointe 'Avis.pdf'
pdf = None
for pl in message.get_payload():
if pl.get_filename() == 'Avis.pdf':
pdf = pl.get_payload(decode=True)
if not pdf:
print('CADA %s: (!) avis introuvable' % cada_id)
continue
# Enregistrement du PDF
pdfname = os.path.join(files_root, 'avis-cada-%s.pdf' % cada_id)
with open(pdfname, 'wb') as outf:
outf.write(pdf)
# Extraction du texte
pro = subprocess.run(['pdftotext', pdfname, '-'],
stdout=subprocess.PIPE,
encoding='utf-8')
# Vérification de la décision
if AVIS_INCOMPETENCE not in pro.stdout:
print('CADA %s: (!) décision inattendue' % cada_id)
# Extraction du député concerné
match = AVIS_RE_DEPUTE.search(pro.stdout)
if not match:
print('CADA %s: (!) nom député introuvable' % cada_id)
else:
nom = match.group(2)
condition = normalize_name(Parlementaire.nom_complet) \
.ilike(normalize_name(nom))
try:
parl = Parlementaire.query.filter(condition).one()
print('CADA %s: %s' % (cada_id, parl.nom_complet))
except NoResultFound:
print('CADA %s: (!) parlementaire inconnu %s'
% (cada_id, nom))
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment