Commit 2fbd0925 by Nicolas Joyard

Gestion de la date de modification

parent 4e159bc4
Showing with 38 additions and 31 deletions
# -*- coding: utf-8 -*-
from datetime import datetime
from collections import defaultdict
import copy
import os
import traceback
from zipfile import ZipFile
......@@ -13,8 +15,8 @@ from ..models import db, Job
class BaseJob(object):
cache = {}
cache_pk = {'themes': 'theme', 'regimes': 'nom'}
cache = defaultdict(lambda: {})
cache_pk = {'themes': 'theme', 'regimes': 'nom', 'jobs': 'nom'}
@property
def job_name(self):
......@@ -23,7 +25,7 @@ class BaseJob(object):
@property
def job(self):
if not self._job:
self._job, _ = self.get_or_create(Job, nom=self.job_name)
self._job = self.save(Job, self.job_name)
return self._job
def __init__(self, app, flush_every=1000):
......@@ -31,8 +33,8 @@ class BaseJob(object):
self.current = None
self._job = None
self._start = datetime.now()
self._count = 0
self._flush_every = 1000
self._created = defaultdict(lambda: 0)
self._updated = defaultdict(lambda: 0)
def debug(self, msg):
self.app.logger.debug(u'<%s> %s' % (self.job_name, msg))
......@@ -54,47 +56,52 @@ class BaseJob(object):
else:
return dateparser.parse(date[0:10])
def get_or_create(self, model, **kwargs):
def save(self, model, pk, data={}):
changed = False
created = False
item = model.query.filter_by(**kwargs).first()
# Search for item in cache
cache = self.cache[model.__tablename__]
pkfield = self.cache_pk.get(model.__tablename__, 'id')
item = cache.get(pk, None)
# Search for item in db
if not item:
if self._count > 0 and self._count % 1000 == 0:
db.session.commit()
item = model.query.filter_by(**{pkfield: pk}).first()
item = model(**kwargs)
if not item:
# Create new item
item = model(**{pkfield: pk})
db.session.add(item)
created = True
self._count += 1
if self._count % self._flush_every == 0:
self.debug('%d objects touched, flushing' % self._count)
db.session.flush()
# Ensure item is now in cache
cache[pk] = item
return item, created
# Update item data
for key, value in data.items():
if getattr(item, key) != value:
changed = True
setattr(item, key, value)
def get_cached(self, model, pk):
item, _ = self.get_cached_(model, pk)
return item
# Update last mod date
if changed or created:
item._last_updated = self._start
def get_cached_(self, model, pk):
if model.__tablename__ not in self.cache:
self.cache[model.__tablename__] = {}
cache = self.cache[model.__tablename__]
pkfield = self.cache_pk.get(model.__tablename__, 'id')
created = False
if pk not in cache:
cache[pk], created = self.get_or_create(model, **{pkfield: pk})
return cache[pk], created
# Update job counters
if created:
self._created[model.__tablename__] += 1
elif changed:
self._updated[model.__tablename__] += 1
return item
def update_status(self, status=None, file=None, filedate=None):
job = self.job
job.date_exec = datetime.now()
job.temps_exec = (datetime.now() - self._start).seconds
job.nb_items = self._count
job.nb_items = sum([v for k, v in self._created.items()]) + \
sum([v for k, v in self._updated.items()])
job.resultat = status or u''
if file:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment