Commit d23a1f68 by Nicolas Joyard

Merge branch 'move_tables', closes !5

parents 45685840 c9c7dd3f
"""Separation tables
Revision ID: 511f685c2fc2
Revises: 14e3a8ca0ec4
Create Date: 2017-04-15 09:17:42.698269
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '511f685c2fc2'
down_revision = '14e3a8ca0ec4'
branch_labels = None
depends_on = None
def upgrade():
op.rename_table('actes', 'an_actes')
op.rename_table('acteurs', 'an_acteurs')
op.rename_table('acteurs_amendements', 'an_acteurs_amendements')
op.rename_table('acteurs_documents', 'an_acteurs_documents')
op.rename_table('acteurs_dossiers', 'an_acteurs_dossiers')
op.rename_table('acteurs_reunions', 'an_acteurs_reunions')
op.rename_table('amendements', 'an_amendements')
op.rename_table('documents', 'an_documents')
op.rename_table('documents_themes', 'an_documents_themes')
op.rename_table('dossiers', 'an_dossiers')
op.rename_table('jobs', 'parlapi_jobs')
op.rename_table('legislatures', 'an_legislatures')
op.rename_table('mandats', 'an_mandats')
op.rename_table('mandats_organes', 'an_mandats_organes')
op.rename_table('odjitems', 'an_odjitems')
op.rename_table('odjpoints', 'an_odjpoints')
op.rename_table('odjpoints_dossiers', 'an_odjpoints_dossiers')
op.rename_table('organes', 'an_organes')
op.rename_table('organes_documents', 'an_organes_documents')
op.rename_table('organes_dossiers', 'an_organes_dossiers')
op.rename_table('organes_reunions', 'an_organes_reunions')
op.rename_table('regimes', 'an_regimes')
op.rename_table('reunions', 'an_reunions')
op.rename_table('scrutins', 'an_scrutins')
op.rename_table('scrutins_groupes', 'an_scrutins_groupes')
op.rename_table('themes', 'an_themes')
op.rename_table('votants', 'an_votants')
def downgrade():
op.rename_table('an_actes', 'actes')
op.rename_table('an_acteurs', 'acteurs')
op.rename_table('an_acteurs_amendements', 'acteurs_amendements')
op.rename_table('an_acteurs_documents', 'acteurs_documents')
op.rename_table('an_acteurs_dossiers', 'acteurs_dossiers')
op.rename_table('an_acteurs_reunions', 'acteurs_reunions')
op.rename_table('an_amendements', 'amendements')
op.rename_table('an_documents', 'documents')
op.rename_table('an_documents_themes', 'documents_themes')
op.rename_table('an_dossiers', 'dossiers')
op.rename_table('an_legislatures', 'legislatures')
op.rename_table('an_mandats', 'mandats')
op.rename_table('an_mandats_organes', 'mandats_organes')
op.rename_table('an_odjitems', 'odjitems')
op.rename_table('an_odjpoints', 'odjpoints')
op.rename_table('an_odjpoints_dossiers', 'odjpoints_dossiers')
op.rename_table('an_organes', 'organes')
op.rename_table('an_organes_documents', 'organes_documents')
op.rename_table('an_organes_dossiers', 'organes_dossiers')
op.rename_table('an_organes_reunions', 'organes_reunions')
op.rename_table('an_regimes', 'regimes')
op.rename_table('an_reunions', 'reunions')
op.rename_table('an_scrutins', 'scrutins')
op.rename_table('an_scrutins_groupes', 'scrutins_groupes')
op.rename_table('an_themes', 'themes')
op.rename_table('an_votants', 'votants')
op.rename_table('parlapi_jobs', 'jobs')
......@@ -26,6 +26,7 @@ def runserver():
# Manage database
#
@manager.command
def createdb():
u'Crée le schéma BDD'
......@@ -50,11 +51,12 @@ manager.add_command('db', MigrateCommand)
# Run import jobs
#
@manager.command
@manager.option('-f', '--file', dest='file', default=None)
def update_amo_an(file=None):
u"Met à jour acteurs, mandats, organes depuis l'AN"
from .jobs.an_amo import run
from .jobs.an.amo import run
app.config.update(SQLALCHEMY_ECHO=False)
run(app, False, file)
......@@ -64,7 +66,7 @@ def update_amo_an(file=None):
@manager.option('-f', '--file', dest='file', default=None)
def update_dossiers_an(file=None):
u"Met à jour dossiers, documents, actes depuis l'AN"
from .jobs.an_dossiers import run
from .jobs.an.dossiers import run
app.config.update(SQLALCHEMY_ECHO=False)
run(app, False, file)
......@@ -74,7 +76,7 @@ def update_dossiers_an(file=None):
@manager.option('-f', '--file', dest='file', default=None)
def update_amendements_an(file=None):
u"Met à jour amendements depuis l'AN"
from .jobs.an_amendements import run
from .jobs.an.amendements import run
app.config.update(SQLALCHEMY_ECHO=False)
run(app, False, file)
......@@ -84,7 +86,7 @@ def update_amendements_an(file=None):
@manager.option('-f', '--file', dest='file', default=None)
def update_scrutins_an(file=None):
u"Met à jour scrutins depuis l'AN"
from .jobs.an_scrutins import run
from .jobs.an.scrutins import run
app.config.update(SQLALCHEMY_ECHO=False)
run(app, False, file)
......@@ -94,7 +96,7 @@ def update_scrutins_an(file=None):
@manager.option('-f', '--file', dest='file', default=None)
def update_reunions_an(file=None):
u"Met à jour réunions depuis l'AN"
from .jobs.an_reunions import run
from .jobs.an.reunions import run
app.config.update(SQLALCHEMY_ECHO=False)
run(app, False, file)
......
......@@ -4,7 +4,7 @@ import graphene
from graphene import relay
from graphene_sqlalchemy import SQLAlchemyObjectType, SQLAlchemyConnectionField
from ..models import (
from ..models.an import (
Acte as ActeModel,
Acteur as ActeurModel,
ActeurAmendement as ActeurAmendementModel,
......@@ -14,7 +14,6 @@ from ..models import (
Amendement as AmendementModel,
Document as DocumentModel,
Dossier as DossierModel,
Job as JobModel,
Legislature as LegislatureModel,
Mandat as MandatModel,
ODJItem as ODJItemModel,
......@@ -30,6 +29,7 @@ from ..models import (
Theme as ThemeModel,
Votant as VotantModel,
)
from ..models.parlapi import Job as JobModel
class Acte(SQLAlchemyObjectType):
......
# -*- coding: utf-8 -*-
from .base import BaseANJob
from .utils import ijson_items
from ..models import Amendement, ActeurAmendement, Document, Legislature, Organe
from ..base import BaseANJob
from ..utils import ijson_items
from ...models.an import (Amendement, ActeurAmendement, Document, Legislature,
Organe)
class ImportAmendementsJob(BaseANJob):
......
# -*- coding: utf-8 -*-
from .base import BaseANJob
from .utils import ijson_items
from ..models import Organe, Regime, Legislature, Acteur, Mandat
from ..base import BaseANJob
from ..utils import ijson_items
from ...models.an import Organe, Regime, Legislature, Acteur, Mandat
class ImportAMOJob(BaseANJob):
......
# -*- coding: utf-8 -*-
from .base import BaseANJob
from .utils import ijson_items
from ..models import (Organe, Legislature, Acteur, Document, Theme, Mandat,
ActeurDocument, OrganeDocument, Dossier, ActeurDossier,
OrganeDossier, Acte)
from ..base import BaseANJob
from ..utils import ijson_items
from ...models.an import (Organe, Legislature, Acteur, Document, Theme, Mandat,
ActeurDocument, OrganeDocument, Dossier, Acte,
OrganeDossier, ActeurDossier)
class ImportDossiersJob(BaseANJob):
......
# -*- coding: utf-8 -*-
from .base import BaseANJob
from .utils import ijson_items
from ..models import (Organe, Acteur, Reunion, ODJItem, ODJPoint,
OrganeReunion, ActeurReunion)
from ..base import BaseANJob
from ..utils import ijson_items
from ...models.an import (Organe, Acteur, Reunion, ODJItem, ODJPoint,
OrganeReunion, ActeurReunion)
class ImportReunionsJob(BaseANJob):
......
# -*- coding: utf-8 -*-
from .base import BaseANJob
from .utils import ijson_items
from ..models import (Organe, Legislature, Acteur, Mandat, Scrutin, Votant,
ScrutinGroupe)
from ..base import BaseANJob
from ..utils import ijson_items
from ...models.an import (Organe, Legislature, Acteur, Mandat, Scrutin, Votant,
ScrutinGroupe)
class ImportScrutinsJob(BaseANJob):
......
......@@ -13,12 +13,17 @@ import requests
from sqlalchemy.inspection import inspect
from ..models import db, Job
from ..models import db
from ..models.parlapi import Job
class BaseJob(object):
cache = defaultdict(lambda: {})
cache_pk = {'themes': 'theme', 'regimes': 'nom', 'jobs': 'nom'}
cache_pk = {
'an_themes': 'theme',
'an_regimes': 'nom',
'parlapi_jobs': 'nom'
}
columns = {}
relations = {}
......
# -*- coding: utf-8 -*-
from .database import db
# -*- coding: utf-8 -*-
from flask_sqlalchemy import SQLAlchemy, BaseQuery
from sqlalchemy_searchable import make_searchable, SearchQueryMixin
db = SQLAlchemy(session_options={'autoflush': False, 'autocommit': False})
make_searchable(options={'regconfig': 'pg_catalog.french'})
class SearchableQuery(BaseQuery, SearchQueryMixin):
pass
# -*- coding: utf-8 -*-
from sqlalchemy_utils.types import TSVectorType
from .database import db, SearchableQuery
#
# Modeles internes
#
class Job(db.Model):
__tablename__ = 'parlapi_jobs'
query_class = SearchableQuery
id = db.Column(db.Integer, primary_key=True)
nom = db.Column(db.Unicode)
date_exec = db.Column(db.DateTime)
url_fichier = db.Column(db.Unicode)
date_fichier = db.Column(db.DateTime)
temps_exec = db.Column(db.Integer)
nb_items = db.Column(db.Integer)
resultat = db.Column(db.Unicode)
search_vector = db.Column(TSVectorType('nom', 'url_fichier'))
......@@ -17,7 +17,8 @@ class API(object):
@property
def summary(self):
descs = {k: {'desc': v.desc, 'count': v.model.query.count()}
descs = {k: {'section': v.section, 'stable': v.stable, 'desc': v.desc,
'count': v.model.query.count()}
for k, v in self.registry.items()
if not v.hidden}
return collections.OrderedDict(sorted(descs.items()))
......@@ -26,7 +27,8 @@ class API(object):
ep = Endpoint(self, self.ma, model, **kwargs)
self.registry[ep.table] = ep
def get_endpoint_or_404(self, table):
def get_endpoint_or_404(self, section, stable):
table = '{}_{}'.format(section, stable)
if table not in self.registry:
abort(404)
return self.registry[table]
......@@ -34,44 +36,57 @@ class API(object):
def setup_routes(self, app, prefix='/rest/'):
prefix = '/%s/' % prefix.strip('/')
@app.route('%s<table>/' % prefix)
def api_list(table):
endpoint = self.get_endpoint_or_404(table)
@app.route('%s<section>/<stable>/' % prefix)
def api_list(section, stable):
endpoint = self.get_endpoint_or_404(section, stable)
return endpoint.list()
@app.route('%s<table>/meta' % prefix)
def api_meta(table):
endpoint = self.get_endpoint_or_404(table)
@app.route('%s<section>/<stable>/meta' % prefix)
def api_meta(section, stable):
endpoint = self.get_endpoint_or_404(section, stable)
return endpoint.describe()
@app.route('%s<table>/<id>' % prefix)
def api_detail(table, id):
endpoint = self.get_endpoint_or_404(table)
@app.route('%s<section>/<stable>/<id>' % prefix)
def api_detail(section, stable, id):
endpoint = self.get_endpoint_or_404(section, stable)
return endpoint.show(id)
@app.route('%s<table>/<id>/<custom>' % prefix)
def api_custom(table, id, custom):
endpoint = self.get_endpoint_or_404(table)
@app.route('%s<section>/<stable>/<id>/<custom>' % prefix)
def api_custom(section, stable, id, custom):
endpoint = self.get_endpoint_or_404(section, stable)
return endpoint.show(id, custom)
def split_table(self, table):
return table.split('_')[0], table.split('_', 1)[1]
def list_url(self, table):
return url_for('api_list', table=table, _external=True)
section, stable = self.split_table(table)
return url_for('api_list', section=section, stable=stable,
_external=True)
def list_url_field(self, table):
return self.ma.AbsoluteURLFor('api_list', table=table)
section, stable = self.split_table(table)
return self.ma.AbsoluteURLFor('api_list', section=section,
stable=stable)
def item_url(self, table, id):
return url_for('api_detail', table, id)
section, stable = self.split_table(table)
return url_for('api_detail', section=section, stable=stable, id=id)
def item_url_field(self, table, attr='id'):
return self.ma.AbsoluteURLFor('api_detail', table=table, id='<%s>' % attr)
section, stable = self.split_table(table)
return self.ma.AbsoluteURLFor('api_detail', section=section,
stable=stable, id='<%s>' % attr)
def relation_url(self, table, id, rel):
return url_for('api_custom', table, id, rel)
section, stable = self.split_table(table)
return url_for('api_custom', section=section, stable=stable, id=id,
custom=rel)
def relation_url_field(self, table, rel):
return self.ma.AbsoluteURLFor('api_custom', table=table, id='<id>',
custom=rel)
section, stable = self.split_table(table)
return self.ma.AbsoluteURLFor('api_custom', section=section,
stable=stable, id='<id>', custom=rel)
def page_url(self, table, page, size, **kwargs):
qs = copy(kwargs)
......
......@@ -48,6 +48,10 @@ class Endpoint(object):
self.model = model
self.table = model.__tablename__
self.section = self.table.split('_')[0]
self.stable = self.table.split('_', 1)[1]
self.desc = description or self.table
self.hidden = hidden
......
# -*- coding: utf-8 -*-
from .api import API
from ..models import (
from ..models.an import (
Acte,
Acteur,
ActeurAmendement,
......@@ -11,7 +11,6 @@ from ..models import (
Amendement,
Document,
Dossier,
Job,
Legislature,
Mandat,
ODJItem,
......@@ -27,9 +26,10 @@ from ..models import (
Theme,
Votant
)
from ..models.parlapi import Job
def setup_an_api(app):
def setup_api(app):
api = API(app)
api.endpoint(
......
......@@ -6,12 +6,11 @@ import os
from flask import render_template, url_for
import humanize
from .models import Job
from .models.parlapi import Job
def setup_routes(app, rest_apis, graphql_api):
for chamber, rest_api in rest_apis.items():
rest_api.setup_routes(app, '/rest/{}/'.format(chamber))
def setup_routes(app, rest_api, graphql_api):
rest_api.setup_routes(app, '/rest/')
graphql_prefix = '/graphql/'
graphql_api.setup_routes(app, graphql_prefix)
......@@ -51,7 +50,11 @@ def setup_routes(app, rest_apis, graphql_api):
return render_template(
'index.html',
piwik=piwik,
rest_apis={ k: v.summary for k, v in rest_apis.items() },
rest_api=rest_api.summary,
sections={
'an': u'Assemblée nationale',
'parlapi': 'ParlAPI'
},
graphql_prefix=graphql_prefix,
jobs=Job.query.all()
)
......@@ -23,8 +23,8 @@ def setup_app(name):
migrate = Migrate(app, db)
# Setup REST API
from .rest.setup import setup_an_api as setup_an_rest_api
an_rest_api = setup_an_rest_api(app)
from .rest.setup import setup_api as setup_rest_api
rest_api = setup_rest_api(app)
# Setup GraphQL API
from .graphql.setup import setup_api as setup_graphql_api
......@@ -32,7 +32,7 @@ def setup_app(name):
# Setup routes
from .routes import setup_routes
setup_routes(app, { 'an': an_rest_api }, graphql_api)
setup_routes(app, rest_api, graphql_api)
# Enable Markdown
Markdown(app)
......
......@@ -63,17 +63,15 @@
<th>Taille</th>
<th>Point d'entrée API</th>
</tr>
{% for chamber, rest_api in rest_apis.items() %}
{% for table, item in rest_api.items() %}
<tr>
<td><img class="chamber-icon" src="{{ url_for('static', filename='chamber-' + chamber + '.png') }}"</td>
<td>{{ item.desc }}</td>
<td>{{ item.count }}</td>
<td>
<a href="/rest/an/{{ table }}"><code>/rest/an/{{ table }}</code></a>
</td>
</tr>
{% endfor %}
{% for table, item in rest_api.items() %}
<tr>
<td><img class="chamber-icon" src="{{ url_for('static', filename='' + item.section + '.png') }}" alt="{{ sections[item.section] }}" title="{{ sections[item.section] }}"></td>
<td>{{ item.desc }}</td>
<td>{{ item.count }}</td>
<td>
<a href="/rest/{{ item.section }}/{{ item.stable }}"><code>/rest/{{ item.section }}/{{ item.stable }}</code></a>
</td>
</tr>
{% endfor %}
</table>
</section>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment