Merge branch 'move_tables', closes !5

migrations/versions/511f685c2fc2_separation_tables.py

+"""Separation tables
+
+Revision ID: 511f685c2fc2
+Revises: 14e3a8ca0ec4
+Create Date: 2017-04-15 09:17:42.698269
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '511f685c2fc2'
+down_revision = '14e3a8ca0ec4'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.rename_table('actes', 'an_actes')
+    op.rename_table('acteurs', 'an_acteurs')
+    op.rename_table('acteurs_amendements', 'an_acteurs_amendements')
+    op.rename_table('acteurs_documents', 'an_acteurs_documents')
+    op.rename_table('acteurs_dossiers', 'an_acteurs_dossiers')
+    op.rename_table('acteurs_reunions', 'an_acteurs_reunions')
+    op.rename_table('amendements', 'an_amendements')
+    op.rename_table('documents', 'an_documents')
+    op.rename_table('documents_themes', 'an_documents_themes')
+    op.rename_table('dossiers', 'an_dossiers')
+    op.rename_table('jobs', 'parlapi_jobs')
+    op.rename_table('legislatures', 'an_legislatures')
+    op.rename_table('mandats', 'an_mandats')
+    op.rename_table('mandats_organes', 'an_mandats_organes')
+    op.rename_table('odjitems', 'an_odjitems')
+    op.rename_table('odjpoints', 'an_odjpoints')
+    op.rename_table('odjpoints_dossiers', 'an_odjpoints_dossiers')
+    op.rename_table('organes', 'an_organes')
+    op.rename_table('organes_documents', 'an_organes_documents')
+    op.rename_table('organes_dossiers', 'an_organes_dossiers')
+    op.rename_table('organes_reunions', 'an_organes_reunions')
+    op.rename_table('regimes', 'an_regimes')
+    op.rename_table('reunions', 'an_reunions')
+    op.rename_table('scrutins', 'an_scrutins')
+    op.rename_table('scrutins_groupes', 'an_scrutins_groupes')
+    op.rename_table('themes', 'an_themes')
+    op.rename_table('votants', 'an_votants')
+
+
+def downgrade():
+    op.rename_table('an_actes', 'actes')
+    op.rename_table('an_acteurs', 'acteurs')
+    op.rename_table('an_acteurs_amendements', 'acteurs_amendements')
+    op.rename_table('an_acteurs_documents', 'acteurs_documents')
+    op.rename_table('an_acteurs_dossiers', 'acteurs_dossiers')
+    op.rename_table('an_acteurs_reunions', 'acteurs_reunions')
+    op.rename_table('an_amendements', 'amendements')
+    op.rename_table('an_documents', 'documents')
+    op.rename_table('an_documents_themes', 'documents_themes')
+    op.rename_table('an_dossiers', 'dossiers')
+    op.rename_table('an_legislatures', 'legislatures')
+    op.rename_table('an_mandats', 'mandats')
+    op.rename_table('an_mandats_organes', 'mandats_organes')
+    op.rename_table('an_odjitems', 'odjitems')
+    op.rename_table('an_odjpoints', 'odjpoints')
+    op.rename_table('an_odjpoints_dossiers', 'odjpoints_dossiers')
+    op.rename_table('an_organes', 'organes')
+    op.rename_table('an_organes_documents', 'organes_documents')
+    op.rename_table('an_organes_dossiers', 'organes_dossiers')
+    op.rename_table('an_organes_reunions', 'organes_reunions')
+    op.rename_table('an_regimes', 'regimes')
+    op.rename_table('an_reunions', 'reunions')
+    op.rename_table('an_scrutins', 'scrutins')
+    op.rename_table('an_scrutins_groupes', 'scrutins_groupes')
+    op.rename_table('an_themes', 'themes')
+    op.rename_table('an_votants', 'votants')
+    op.rename_table('parlapi_jobs', 'jobs')

parlapi/cli.py

@@ -26,6 +26,7 @@ def runserver():
 # Manage database
 #
 
+
 @manager.command
 def createdb():
     u'Crée le schéma BDD'
@@ -50,11 +51,12 @@ manager.add_command('db', MigrateCommand)
 # Run import jobs
 #
 
+
 @manager.command
 @manager.option('-f', '--file', dest='file', default=None)
 def update_amo_an(file=None):
     u"Met à jour acteurs, mandats, organes depuis l'AN"
-    from .jobs.an_amo import run
+    from .jobs.an.amo import run
 
     app.config.update(SQLALCHEMY_ECHO=False)
     run(app, False, file)
@@ -64,7 +66,7 @@ def update_amo_an(file=None):
 @manager.option('-f', '--file', dest='file', default=None)
 def update_dossiers_an(file=None):
     u"Met à jour dossiers, documents, actes depuis l'AN"
-    from .jobs.an_dossiers import run
+    from .jobs.an.dossiers import run
 
     app.config.update(SQLALCHEMY_ECHO=False)
     run(app, False, file)
@@ -74,7 +76,7 @@ def update_dossiers_an(file=None):
 @manager.option('-f', '--file', dest='file', default=None)
 def update_amendements_an(file=None):
     u"Met à jour amendements depuis l'AN"
-    from .jobs.an_amendements import run
+    from .jobs.an.amendements import run
 
     app.config.update(SQLALCHEMY_ECHO=False)
     run(app, False, file)
@@ -84,7 +86,7 @@ def update_amendements_an(file=None):
 @manager.option('-f', '--file', dest='file', default=None)
 def update_scrutins_an(file=None):
     u"Met à jour scrutins depuis l'AN"
-    from .jobs.an_scrutins import run
+    from .jobs.an.scrutins import run
 
     app.config.update(SQLALCHEMY_ECHO=False)
     run(app, False, file)
@@ -94,7 +96,7 @@ def update_scrutins_an(file=None):
 @manager.option('-f', '--file', dest='file', default=None)
 def update_reunions_an(file=None):
     u"Met à jour réunions depuis l'AN"
-    from .jobs.an_reunions import run
+    from .jobs.an.reunions import run
 
     app.config.update(SQLALCHEMY_ECHO=False)
     run(app, False, file)

parlapi/graphql/schema.py

@@ -4,7 +4,7 @@ import graphene
 from graphene import relay
 from graphene_sqlalchemy import SQLAlchemyObjectType, SQLAlchemyConnectionField
 
-from ..models import (
+from ..models.an import (
     Acte as ActeModel,
     Acteur as ActeurModel,
     ActeurAmendement as ActeurAmendementModel,
@@ -14,7 +14,6 @@ from ..models import (
     Amendement as AmendementModel,
     Document as DocumentModel,
     Dossier as DossierModel,
-    Job as JobModel,
     Legislature as LegislatureModel,
     Mandat as MandatModel,
     ODJItem as ODJItemModel,
@@ -30,6 +29,7 @@ from ..models import (
     Theme as ThemeModel,
     Votant as VotantModel,
 )
+from ..models.parlapi import Job as JobModel
 
 
 class Acte(SQLAlchemyObjectType):

parlapi/jobs/an_amendements.py → parlapi/jobs/an/amendements.py

 # -*- coding: utf-8 -*-
 
-from .base import BaseANJob
-from .utils import ijson_items
-from ..models import Amendement, ActeurAmendement, Document, Legislature, Organe
+from ..base import BaseANJob
+from ..utils import ijson_items
+from ...models.an import (Amendement, ActeurAmendement, Document, Legislature,
+                         Organe)
 
 
 class ImportAmendementsJob(BaseANJob):

parlapi/jobs/an_amo.py → parlapi/jobs/an/amo.py

 # -*- coding: utf-8 -*-
 
-from .base import BaseANJob
-from .utils import ijson_items
-from ..models import Organe, Regime, Legislature, Acteur, Mandat
+from ..base import BaseANJob
+from ..utils import ijson_items
+from ...models.an import Organe, Regime, Legislature, Acteur, Mandat
 
 
 class ImportAMOJob(BaseANJob):

parlapi/jobs/an_dossiers.py → parlapi/jobs/an/dossiers.py

 # -*- coding: utf-8 -*-
 
-from .base import BaseANJob
-from .utils import ijson_items
-from ..models import (Organe, Legislature, Acteur, Document, Theme, Mandat,
-                      ActeurDocument, OrganeDocument, Dossier, ActeurDossier,
-                      OrganeDossier, Acte)
+from ..base import BaseANJob
+from ..utils import ijson_items
+from ...models.an import (Organe, Legislature, Acteur, Document, Theme, Mandat,
+                         ActeurDocument, OrganeDocument, Dossier, Acte,
+                         OrganeDossier, ActeurDossier)
 
 
 class ImportDossiersJob(BaseANJob):

parlapi/jobs/an_reunions.py → parlapi/jobs/an/reunions.py

 # -*- coding: utf-8 -*-
 
-from .base import BaseANJob
-from .utils import ijson_items
-from ..models import (Organe, Acteur, Reunion, ODJItem, ODJPoint,
-                      OrganeReunion, ActeurReunion)
+from ..base import BaseANJob
+from ..utils import ijson_items
+from ...models.an import (Organe, Acteur, Reunion, ODJItem, ODJPoint,
+                         OrganeReunion, ActeurReunion)
 
 
 class ImportReunionsJob(BaseANJob):

parlapi/jobs/an_scrutins.py → parlapi/jobs/an/scrutins.py

 # -*- coding: utf-8 -*-
 
-from .base import BaseANJob
-from .utils import ijson_items
-from ..models import (Organe, Legislature, Acteur, Mandat, Scrutin, Votant,
-                      ScrutinGroupe)
+from ..base import BaseANJob
+from ..utils import ijson_items
+from ...models.an import (Organe, Legislature, Acteur, Mandat, Scrutin, Votant,
+                         ScrutinGroupe)
 
 
 class ImportScrutinsJob(BaseANJob):

parlapi/jobs/base.py

@@ -13,12 +13,17 @@ import requests
 
 from sqlalchemy.inspection import inspect
 
-from ..models import db, Job
+from ..models import db
+from ..models.parlapi import Job
 
 
 class BaseJob(object):
     cache = defaultdict(lambda: {})
-    cache_pk = {'themes': 'theme', 'regimes': 'nom', 'jobs': 'nom'}
+    cache_pk = {
+        'an_themes': 'theme',
+        'an_regimes': 'nom',
+        'parlapi_jobs': 'nom'
+    }
     columns = {}
     relations = {}
 

parlapi/models/__init__.py

+# -*- coding: utf-8 -*-
+
+from .database import db

parlapi/models/database.py

+# -*- coding: utf-8 -*-
+
+from flask_sqlalchemy import SQLAlchemy, BaseQuery
+from sqlalchemy_searchable import make_searchable, SearchQueryMixin
+
+db = SQLAlchemy(session_options={'autoflush': False, 'autocommit': False})
+make_searchable(options={'regconfig': 'pg_catalog.french'})
+
+
+class SearchableQuery(BaseQuery, SearchQueryMixin):
+    pass

parlapi/models/parlapi.py

+# -*- coding: utf-8 -*-
+
+from sqlalchemy_utils.types import TSVectorType
+
+from .database import db, SearchableQuery
+
+
+#
+# Modeles internes
+#
+
+class Job(db.Model):
+    __tablename__ = 'parlapi_jobs'
+    query_class = SearchableQuery
+
+    id = db.Column(db.Integer, primary_key=True)
+    nom = db.Column(db.Unicode)
+    date_exec = db.Column(db.DateTime)
+    url_fichier = db.Column(db.Unicode)
+    date_fichier = db.Column(db.DateTime)
+    temps_exec = db.Column(db.Integer)
+    nb_items = db.Column(db.Integer)
+    resultat = db.Column(db.Unicode)
+
+    search_vector = db.Column(TSVectorType('nom', 'url_fichier'))
+

parlapi/rest/api.py

@@ -17,7 +17,8 @@ class API(object):
 
     @property
     def summary(self):
-        descs = {k: {'desc': v.desc, 'count': v.model.query.count()}
+        descs = {k: {'section': v.section, 'stable': v.stable, 'desc': v.desc,
+                     'count': v.model.query.count()}
                  for k, v in self.registry.items()
                  if not v.hidden}
         return collections.OrderedDict(sorted(descs.items()))
@@ -26,7 +27,8 @@ class API(object):
         ep = Endpoint(self, self.ma, model, **kwargs)
         self.registry[ep.table] = ep
 
-    def get_endpoint_or_404(self, table):
+    def get_endpoint_or_404(self, section, stable):
+        table = '{}_{}'.format(section, stable)
         if table not in self.registry:
             abort(404)
         return self.registry[table]
@@ -34,44 +36,57 @@ class API(object):
     def setup_routes(self, app, prefix='/rest/'):
         prefix = '/%s/' % prefix.strip('/')
 
-        @app.route('%s<table>/' % prefix)
-        def api_list(table):
-            endpoint = self.get_endpoint_or_404(table)
+        @app.route('%s<section>/<stable>/' % prefix)
+        def api_list(section, stable):
+            endpoint = self.get_endpoint_or_404(section, stable)
             return endpoint.list()
 
-        @app.route('%s<table>/meta' % prefix)
-        def api_meta(table):
-            endpoint = self.get_endpoint_or_404(table)
+        @app.route('%s<section>/<stable>/meta' % prefix)
+        def api_meta(section, stable):
+            endpoint = self.get_endpoint_or_404(section, stable)
             return endpoint.describe()
 
-        @app.route('%s<table>/<id>' % prefix)
-        def api_detail(table, id):
-            endpoint = self.get_endpoint_or_404(table)
+        @app.route('%s<section>/<stable>/<id>' % prefix)
+        def api_detail(section, stable, id):
+            endpoint = self.get_endpoint_or_404(section, stable)
             return endpoint.show(id)
 
-        @app.route('%s<table>/<id>/<custom>' % prefix)
-        def api_custom(table, id, custom):
-            endpoint = self.get_endpoint_or_404(table)
+        @app.route('%s<section>/<stable>/<id>/<custom>' % prefix)
+        def api_custom(section, stable, id, custom):
+            endpoint = self.get_endpoint_or_404(section, stable)
             return endpoint.show(id, custom)
 
+    def split_table(self, table):
+        return table.split('_')[0], table.split('_', 1)[1]
+
     def list_url(self, table):
-        return url_for('api_list', table=table, _external=True)
+        section, stable = self.split_table(table)
+        return url_for('api_list', section=section, stable=stable,
+                       _external=True)
 
     def list_url_field(self, table):
-        return self.ma.AbsoluteURLFor('api_list', table=table)
+        section, stable = self.split_table(table)
+        return self.ma.AbsoluteURLFor('api_list', section=section,
+                                      stable=stable)
 
     def item_url(self, table, id):
-        return url_for('api_detail', table, id)
+        section, stable = self.split_table(table)
+        return url_for('api_detail', section=section, stable=stable, id=id)
 
     def item_url_field(self, table, attr='id'):
-        return self.ma.AbsoluteURLFor('api_detail', table=table, id='<%s>' % attr)
+        section, stable = self.split_table(table)
+        return self.ma.AbsoluteURLFor('api_detail', section=section,
+                                      stable=stable, id='<%s>' % attr)
 
     def relation_url(self, table, id, rel):
-        return url_for('api_custom', table, id, rel)
+        section, stable = self.split_table(table)
+        return url_for('api_custom', section=section, stable=stable, id=id,
+                       custom=rel)
 
     def relation_url_field(self, table, rel):
-        return self.ma.AbsoluteURLFor('api_custom', table=table, id='<id>',
-                                      custom=rel)
+        section, stable = self.split_table(table)
+        return self.ma.AbsoluteURLFor('api_custom', section=section,
+                                      stable=stable, id='<id>', custom=rel)
 
     def page_url(self, table, page, size, **kwargs):
         qs = copy(kwargs)

parlapi/rest/endpoint.py

@@ -48,6 +48,10 @@ class Endpoint(object):
 
         self.model = model
         self.table = model.__tablename__
+
+        self.section = self.table.split('_')[0]
+        self.stable = self.table.split('_', 1)[1]
+
         self.desc = description or self.table
         self.hidden = hidden
 

parlapi/rest/setup.py

 # -*- coding: utf-8 -*-
 from .api import API
 
-from ..models import (
+from ..models.an import (
     Acte,
     Acteur,
     ActeurAmendement,
@@ -11,7 +11,6 @@ from ..models import (
     Amendement,
     Document,
     Dossier,
-    Job,
     Legislature,
     Mandat,
     ODJItem,
@@ -27,9 +26,10 @@ from ..models import (
     Theme,
     Votant
 )
+from ..models.parlapi import Job
 
 
-def setup_an_api(app):
+def setup_api(app):
     api = API(app)
 
     api.endpoint(

parlapi/routes.py

@@ -6,12 +6,11 @@ import os
 from flask import render_template, url_for
 import humanize
 
-from .models import Job
+from .models.parlapi import Job
 
 
-def setup_routes(app, rest_apis, graphql_api):
-    for chamber, rest_api in rest_apis.items():
-        rest_api.setup_routes(app, '/rest/{}/'.format(chamber))
+def setup_routes(app, rest_api, graphql_api):
+    rest_api.setup_routes(app, '/rest/')
 
     graphql_prefix = '/graphql/'
     graphql_api.setup_routes(app, graphql_prefix)
@@ -51,7 +50,11 @@ def setup_routes(app, rest_apis, graphql_api):
         return render_template(
             'index.html',
             piwik=piwik,
-            rest_apis={ k: v.summary for k, v in rest_apis.items() },
+            rest_api=rest_api.summary,
+            sections={
+                'an': u'Assemblée nationale',
+                'parlapi': 'ParlAPI'
+            },
             graphql_prefix=graphql_prefix,
             jobs=Job.query.all()
         )

parlapi/setup_app.py

@@ -23,8 +23,8 @@ def setup_app(name):
     migrate = Migrate(app, db)
 
     # Setup REST API
-    from .rest.setup import setup_an_api as setup_an_rest_api
-    an_rest_api = setup_an_rest_api(app)
+    from .rest.setup import setup_api as setup_rest_api
+    rest_api = setup_rest_api(app)
 
     # Setup GraphQL API
     from .graphql.setup import setup_api as setup_graphql_api
@@ -32,7 +32,7 @@ def setup_app(name):
 
     # Setup routes
     from .routes import setup_routes
-    setup_routes(app, { 'an': an_rest_api }, graphql_api)
+    setup_routes(app, rest_api, graphql_api)
 
     # Enable Markdown
     Markdown(app)

parlapi/templates/index.html

@@ -63,17 +63,15 @@
             <th>Taille</th>
             <th>Point d'entrée API</th>
           </tr>
-          {% for chamber, rest_api in rest_apis.items() %}
-            {% for table, item in rest_api.items() %}
-              <tr>
-                <td><img class="chamber-icon" src="{{ url_for('static', filename='chamber-' + chamber + '.png') }}"</td>
-                <td>{{ item.desc }}</td>
-                <td>{{ item.count }}</td>
-                <td>
-                  <a href="/rest/an/{{ table }}"><code>/rest/an/{{ table }}</code></a>
-                </td>
-              </tr>
-            {% endfor %}
+          {% for table, item in rest_api.items() %}
+            <tr>
+              <td><img class="chamber-icon" src="{{ url_for('static', filename='' + item.section + '.png') }}" alt="{{ sections[item.section] }}" title="{{ sections[item.section] }}"></td>
+              <td>{{ item.desc }}</td>
+              <td>{{ item.count }}</td>
+              <td>
+                <a href="/rest/{{ item.section }}/{{ item.stable }}"><code>/rest/{{ item.section }}/{{ item.stable }}</code></a>
+              </td>
+            </tr>
           {% endfor %}
         </table>
       </section>