From 78fbc7a03f983d70db92279822fb8cb5c7670010 Mon Sep 17 00:00:00 2001 From: Alex Meadows Date: Tue, 4 Jun 2019 15:14:04 -0400 Subject: [PATCH 1/2] process_tracker_python-30 Extract Lookups should return Extract objects All the extract file finders now return extract objects so they can be modified as they are used. A helper method was also added that provides the full filename of a given extract (called full_filepath). Also realized that I missed adding string representations of a bunch of the data models. Closes #30 --- process_tracker/models/actor.py | 4 ++++ process_tracker/models/extract.py | 36 +++++++++++++++++++++++++++++- process_tracker/process_tracker.py | 35 +++++++++++------------------ tests/test_process_tracker.py | 15 +++++++++---- 4 files changed, 63 insertions(+), 27 deletions(-) diff --git a/process_tracker/models/actor.py b/process_tracker/models/actor.py index 7e81932..aff6f4d 100755 --- a/process_tracker/models/actor.py +++ b/process_tracker/models/actor.py @@ -11,3 +11,7 @@ class Actor(Base): actor_id = Column(Integer, Sequence('actor_lkup_actor_id_seq'), primary_key=True) actor_name = Column(String(250), nullable=False, unique=True) + + def __repr__(self): + return "" % (self.actor_id + , self.actor_name) diff --git a/process_tracker/models/extract.py b/process_tracker/models/extract.py index 963cbb9..fdf672c 100755 --- a/process_tracker/models/extract.py +++ b/process_tracker/models/extract.py @@ -2,6 +2,7 @@ # Models for Extract (Data) entities from datetime import datetime +from os.path import join from sqlalchemy import Column, DateTime, ForeignKey, Integer, Sequence, String from sqlalchemy.orm import relationship @@ -18,6 +19,11 @@ class ExtractStatus(Base): extracts = relationship("ExtractProcess") + def __repr__(self): + + return "" % (self.extract_status_id + , self.extract_status_name) + class Extract(Base): @@ -32,6 +38,17 @@ class Extract(Base): extract_process = relationship("ExtractProcess", back_populates='process_extracts') locations = relationship("Location", foreign_keys=[extract_location_id]) + def __repr__(self): + + return "" % (self.extract_id + , self.extract_filename + , self.extract_location_id + , self.extract_status_id) + + def full_filepath(self): + + return join(self.locations.location_path, self.extract_filename) + class ExtractProcess(Base): @@ -45,6 +62,12 @@ class ExtractProcess(Base): process_extracts = relationship('Extract', foreign_keys=[extract_tracking_id]) extract_processes = relationship('ProcessTracking', foreign_keys=[process_tracking_id]) + def __repr__(self): + + return "" % (self.extract_tracking_id + , self.process_tracking_id + , self.extract_process_status_id) + class LocationType(Base): @@ -54,7 +77,12 @@ class LocationType(Base): location_type_name = Column(String(25), unique=True, nullable=False) locations = relationship('Location', back_populates='location_types') - + + def __repr__(self): + + return "" % (self.location_type_id + , self.location_type_name) + class Location(Base): @@ -68,3 +96,9 @@ class Location(Base): extracts = relationship("Extract") location_types = relationship('LocationType', foreign_keys=[location_type]) + + def __repr__(self): + + return "" % (self.location_id + , self.location_name + , self.location_path) \ No newline at end of file diff --git a/process_tracker/process_tracker.py b/process_tracker/process_tracker.py index 7af97d5..c5ae735 100755 --- a/process_tracker/process_tracker.py +++ b/process_tracker/process_tracker.py @@ -4,7 +4,6 @@ from datetime import datetime import logging import os -from os.path import join from sqlalchemy.orm import aliased @@ -110,20 +109,16 @@ def find_ready_extracts_by_filename(self, filename): :param filename: :return: """ - extract_files = [] - process_files = self.session.query(Extract.extract_filename, Location.location_path)\ - .join(Location)\ + process_files = self.session.query(Extract)\ .join(ExtractStatus)\ .filter(Extract.extract_filename.like("%" + filename + "%"))\ .filter(ExtractStatus.extract_status_name == 'ready') \ .order_by(Extract.extract_registration_date_time)\ - .order_by(Extract.extract_id) + .order_by(Extract.extract_id)\ + .all() - for record in process_files: - extract_files.append(join(record.location_path, record.extract_filename)) - - return extract_files + return process_files def find_ready_extracts_by_location(self, location): """ @@ -131,28 +126,24 @@ def find_ready_extracts_by_location(self, location): :param location: :return: """ - extract_files = [] - process_files = self.session.query(Extract.extract_filename, Location.location_path)\ + process_files = self.session.query(Extract)\ .join(Location)\ .join(ExtractStatus)\ .filter(ExtractStatus.extract_status_name == 'ready')\ .filter(Location.location_name == location) \ - .order_by(Extract.extract_registration_date_time) - - for record in process_files: - extract_files.append(join(record.location_path, record.extract_filename)) + .order_by(Extract.extract_registration_date_time)\ + .all() - return extract_files + return process_files def find_ready_extracts_by_process(self, extract_process_name): """ For the given named process, find the extracts that are ready for processing. :return: List of OS specific filepaths with filenames. """ - extract_files = [] - process_files = self.session.query(Extract.extract_filename, Location.location_path) \ + process_files = self.session.query(Extract) \ .join(ExtractStatus, Extract.extract_status_id == ExtractStatus.extract_status_id) \ .join(Location, Extract.extract_location_id == Location.location_id) \ .join(ExtractProcess, Extract.extract_id == ExtractProcess.extract_tracking_id) \ @@ -160,12 +151,12 @@ def find_ready_extracts_by_process(self, extract_process_name): .join(Process) \ .filter(Process.process_name == extract_process_name , ExtractStatus.extract_status_name == 'ready') \ - .order_by(Extract.extract_registration_date_time) + .order_by(Extract.extract_registration_date_time)\ + .all() - for record in process_files: - extract_files.append(join(record.location_path, record.extract_filename)) + self.logger.info('Returning extract files by process.') - return extract_files + return process_files def get_latest_tracking_record(self, process): """ diff --git a/tests/test_process_tracker.py b/tests/test_process_tracker.py index b5a04bf..639413d 100755 --- a/tests/test_process_tracker.py +++ b/tests/test_process_tracker.py @@ -129,8 +129,9 @@ def test_find_ready_extracts_by_filename_full(self): expected_result = ['/home/test/extract_dir/test_extract_filename2.csv'] given_result = self.process_tracker.find_ready_extracts_by_filename('test_extract_filename2.csv') + given_result = [record.full_filepath() for record in given_result] - self.assertEqual(expected_result, given_result) + self.assertCountEqual(expected_result, given_result) def test_find_ready_extracts_by_filename_partial(self): """ @@ -161,8 +162,9 @@ def test_find_ready_extracts_by_filename_partial(self): , '/home/test/extract_dir/test_extract_filename3-2.csv'] given_result = self.process_tracker.find_ready_extracts_by_filename('test_extract_filename') + given_result = [record.full_filepath() for record in given_result] - self.assertEqual(expected_result, given_result) + self.assertCountEqual(expected_result, given_result) def test_find_ready_extracts_by_filename_partial_not_descending(self): """ @@ -193,6 +195,7 @@ def test_find_ready_extracts_by_filename_partial_not_descending(self): , '/home/test/extract_dir/test_extract_filename3-1.csv'] given_result = self.process_tracker.find_ready_extracts_by_filename('test_extract_filename') + given_result = [record.full_filepath() for record in given_result] self.assertNotEqual(expected_result, given_result) @@ -225,8 +228,9 @@ def test_find_ready_extracts_by_location(self): , '/home/test/extract_dir/test_extract_filename4-2.csv'] given_result = self.process_tracker.find_ready_extracts_by_location('Test Location') + given_result = [record.full_filepath() for record in given_result] - self.assertEqual(expected_result, given_result) + self.assertCountEqual(expected_result, given_result) def test_find_ready_extracts_by_location_not_descending(self): """ @@ -257,6 +261,7 @@ def test_find_ready_extracts_by_location_not_descending(self): , '/home/test/extract_dir/test_extract_filename4-1.csv'] given_result = self.process_tracker.find_ready_extracts_by_location('Test Location') + given_result = [record.full_filepath() for record in given_result] self.assertNotEqual(expected_result, given_result) @@ -288,8 +293,9 @@ def test_find_ready_extracts_by_process(self): , '/home/test/extract_dir/test_extract_filename5-2.csv'] given_result = self.process_tracker.find_ready_extracts_by_process('Testing Process Tracking Initialization') + given_result = [record.full_filepath() for record in given_result] - self.assertEqual(sorted(expected_result), sorted(given_result)) + self.assertCountEqual(expected_result, given_result) def test_find_ready_extracts_by_process_not_descending(self): """ @@ -323,6 +329,7 @@ def test_find_ready_extracts_by_process_not_descending(self): , '/home/test/extract_dir/test_extract_filename5-1.csv'] given_result = self.process_tracker.find_ready_extracts_by_process('Testing Process Tracking Initialization') + given_result = [record.full_filepath() for record in given_result] self.assertNotEqual(expected_result, given_result) From c16e0a0c08288672cc6d0a5f407ed1f707faf23a Mon Sep 17 00:00:00 2001 From: Alex Meadows Date: Tue, 4 Jun 2019 15:34:00 -0400 Subject: [PATCH 2/2] process_tracker_python-30 Extract Lookups should return Extract objects All the extract file finders now return extract objects so they can be modified as they are used. A helper method was also added that provides the full filename of a given extract (called full_filepath). Also realized that I missed adding string representations of a bunch of the data models. --- tests/test_process_tracker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_process_tracker.py b/tests/test_process_tracker.py index a9d7fd0..91dc7da 100755 --- a/tests/test_process_tracker.py +++ b/tests/test_process_tracker.py @@ -431,8 +431,8 @@ def test_register_new_process_run_exception(self): with self.assertRaises(Exception) as context: # Running registration a second time to mimic job being run twice self.process_tracker.register_new_process_run() - - return self.assertTrue('Process Testing Process Tracking Initialization ' + print(context.exception) + return self.assertTrue('The process Testing Process Tracking Initialization ' 'is currently running.' in str(context.exception)) def test_register_new_process_run_with_previous_run(self):