Commit 4d86c6ea by Nicolas Joyard

Fix last_modified + conversion dates

parent 246b8445
Showing with 35 additions and 4 deletions
......@@ -11,12 +11,16 @@ from bs4 import BeautifulSoup
import dateparser
import requests
from sqlalchemy.inspection import inspect
from ..models import db, Job
class BaseJob(object):
cache = defaultdict(lambda: {})
cache_pk = {'themes': 'theme', 'regimes': 'nom', 'jobs': 'nom'}
columns = {}
relations = {}
@property
def job_name(self):
......@@ -60,6 +64,15 @@ class BaseJob(object):
changed = False
created = False
# Get column info
if model.__tablename__ not in self.columns:
mapper = inspect(model)
self.columns[model.__tablename__] = { c.name: type(c.type).__name__ for c in mapper.columns }
# self.relations[model.__tablename__] = mapper.relationships
columns = self.columns[model.__tablename__]
# relations = self.relations[model.__tablename__]
# Search for item in cache
cache = self.cache[model.__tablename__]
pkfield = self.cache_pk.get(model.__tablename__, 'id')
......@@ -71,6 +84,8 @@ class BaseJob(object):
if not item:
# Create new item
# self.debug(u'{} #{} created'.format(model.__tablename__, pk))
item = model(**{pkfield: pk})
db.session.add(item)
created = True
......@@ -79,14 +94,30 @@ class BaseJob(object):
cache[pk] = item
# Update item data
for key, value in data.items():
if getattr(item, key) != value:
for key, newvalue in data.items():
curvalue = getattr(item, key)
if key in columns:
if columns[key] == 'Date' and isinstance(newvalue, datetime):
newvalue = newvalue.date()
if curvalue != newvalue:
# self.debug(
# u"{} #{} changed {}: {} => {}".format(
# model.__tablename__,
# pk,
# key,
# curvalue,
# newvalue
# )
# )
changed = True
setattr(item, key, value)
setattr(item, key, newvalue)
# Update last mod date
if changed or created:
item._last_updated = self._start
item._last_modified = self._start
# Update job counters
if created:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment