From 46c57a7d1ac3ea3035cf35d872a06dc6f599b454 Mon Sep 17 00:00:00 2001 From: Alex Meadows Date: Thu, 19 Dec 2019 10:08:27 -0500 Subject: [PATCH 1/5] process_tracker_python-144 Add Process Dependency Classifier :sparkles: Dependency type added Extract and Process dependencies can now be classified by type. --- dbscripts/mysql_process_tracker.sql | 23 ++++++++++++++-- dbscripts/mysql_process_tracker_defaults.sql | 3 ++- dbscripts/postgresql_process_tracker.sql | 23 ++++++++++++++++ .../postgresql_process_tracker_defaults.sql | 6 ++++- process_tracker/models/extract.py | 6 +++++ process_tracker/models/process.py | 26 +++++++++++++++++++ 6 files changed, 83 insertions(+), 4 deletions(-) diff --git a/dbscripts/mysql_process_tracker.sql b/dbscripts/mysql_process_tracker.sql index d353f11..f2cf0c4 100644 --- a/dbscripts/mysql_process_tracker.sql +++ b/dbscripts/mysql_process_tracker.sql @@ -1,5 +1,18 @@ USE process_tracker; +create table process_tracker.dependency_type_lkup +( + dependency_type_id int auto_increment + primary key, + dependency_type_name varchar(75) not null, + created_date_time timestamp default CURRENT_TIMESTAMP not null, + created_by int default 0 not null, + update_date_time timestamp default CURRENT_TIMESTAMP not null on update CURRENT_TIMESTAMP, + updated_by int default 0 not null, + constraint dependency_type_lkup_dependency_type_name_uindex + unique (dependency_type_name) +); + create table process_tracker.character_set_lkup ( character_set_id int auto_increment @@ -253,6 +266,7 @@ create table process_tracker.extract_dependency ( parent_extract_id int not null, child_extract_id int not null, + dependency_type_id int default 0 not null, created_date_time timestamp default CURRENT_TIMESTAMP not null, created_by int default 0 not null, update_date_time timestamp default CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP not null, @@ -261,7 +275,9 @@ create table process_tracker.extract_dependency constraint extract_dependency_fk01 foreign key (parent_extract_id) references extract_tracking (extract_id), constraint extract_dependency_fk02 - foreign key (child_extract_id) references extract_tracking (extract_id) + foreign key (child_extract_id) references extract_tracking (extract_id), + constraint extract_dependency_fk03 + foreign key (dependency_type_id) references dependency_type_lkup (dependency_type_id) ) comment 'Table tracking interdependencies between extract files.'; @@ -373,6 +389,7 @@ create table process_dependency ( parent_process_id int not null, child_process_id int not null, + dependency_type_id int default 0 not null, created_date_time timestamp default CURRENT_TIMESTAMP not null, created_by int default 0 not null, update_date_time timestamp default CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP not null, @@ -381,7 +398,9 @@ create table process_dependency constraint process_dependency_ibfk_1 foreign key (parent_process_id) references process (process_id), constraint process_dependency_ibfk_2 - foreign key (child_process_id) references process (process_id) + foreign key (child_process_id) references process (process_id), + constraint process_dependency_fk03 + foreign key (dependency_type_id) references dependency_type_lkup (dependency_type_id) ); create index child_process_id diff --git a/dbscripts/mysql_process_tracker_defaults.sql b/dbscripts/mysql_process_tracker_defaults.sql index 9e5e927..0c0ad18 100644 --- a/dbscripts/mysql_process_tracker_defaults.sql +++ b/dbscripts/mysql_process_tracker_defaults.sql @@ -53,4 +53,5 @@ INSERT INTO process_tracker.filesize_type_lkup (filesize_type_id, filesize_type_ INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (1, 'Undefined'); INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (2, 'Database'); INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (3, 'Internal'); -INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (4, 'External'); \ No newline at end of file +INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (4, 'External'); + diff --git a/dbscripts/postgresql_process_tracker.sql b/dbscripts/postgresql_process_tracker.sql index c004e97..1fccd09 100644 --- a/dbscripts/postgresql_process_tracker.sql +++ b/dbscripts/postgresql_process_tracker.sql @@ -7,6 +7,21 @@ create schema process_tracker; alter schema process_tracker owner to pt_admin; +create table process_tracker.dependency_type_lkup +( + dependency_type_id serial not null + constraint dependency_type_lkup_pk + primary key, + dependency_type_name varchar(75) not null, + created_date_time timestamptz default CURRENT_TIMESTAMP not null, + created_by int default 0 not null, + update_date_time timestamptz default CURRENT_TIMESTAMP not null, + updated_by int default 0 not null +); + +create unique index dependency_type_lkup_dependency_type_name_uindex + on process_tracker.dependency_type_lkup (dependency_type_name); + create table dataset_type_lkup ( dataset_type_id serial not null @@ -406,6 +421,9 @@ create table process_dependency references process, constraint process_dependency_pk primary key (child_process_id, parent_process_id), + dependency_type_id int default 0 not null + constraint process_dependency_fk03 + references dependency_type_lkup, created_date_time timestamp with time zone default CURRENT_TIMESTAMP not null, created_by integer default 0 not null, update_date_time timestamp with time zone default CURRENT_TIMESTAMP not null, @@ -1036,6 +1054,9 @@ create table extract_dependency child_extract_id integer not null constraint extract_dependency_fk02 references extract_tracking, + dependency_type_id int default 0 not null + constraint extract_dependency_fk03 + references dependency_type_lkup, constraint extract_dependency_pk primary key (parent_extract_id, child_extract_id), created_date_time timestamp with time zone default CURRENT_TIMESTAMP not null, @@ -1136,6 +1157,8 @@ CREATE TRIGGER data_type_lkup_update_date_time_trg BEFORE UPDATE ON process_tracker.data_type_lkup FOR EACH ROW EXECUTE PROCEDURE update_date_time_trigger(); CREATE TRIGGER dataset_type_lkup_update_date_time_trg BEFORE UPDATE ON process_tracker.dataset_type_lkup FOR EACH ROW EXECUTE PROCEDURE update_date_time_trigger(); +CREATE TRIGGER dependency_type_lkup_update_date_time_trg BEFORE UPDATE + ON process_tracker.dependency_type_lkup FOR EACH ROW EXECUTE PROCEDURE update_date_time_trigger(); CREATE TRIGGER error_tracking_update_date_time_trg BEFORE UPDATE ON process_tracker.error_tracking FOR EACH ROW EXECUTE PROCEDURE update_date_time_trigger(); CREATE TRIGGER error_type_lkup_update_date_time_trg BEFORE UPDATE diff --git a/dbscripts/postgresql_process_tracker_defaults.sql b/dbscripts/postgresql_process_tracker_defaults.sql index cf2efde..26a62f3 100644 --- a/dbscripts/postgresql_process_tracker_defaults.sql +++ b/dbscripts/postgresql_process_tracker_defaults.sql @@ -52,4 +52,8 @@ INSERT INTO process_tracker.filesize_type_lkup (filesize_type_id, filesize_type_ INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (1, 'Undefined'); INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (2, 'Database'); INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (3, 'Internal'); -INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (4, 'External'); \ No newline at end of file +INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (4, 'External'); + +INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (0, 'Undefined'); +INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (1, 'Hard'); +INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (2, 'Soft'); \ No newline at end of file diff --git a/process_tracker/models/extract.py b/process_tracker/models/extract.py index ae9f69a..169ae19 100755 --- a/process_tracker/models/extract.py +++ b/process_tracker/models/extract.py @@ -211,6 +211,12 @@ class ExtractDependency(Base, BaseColumn): primary_key=True, nullable=False, ) + dependency_type_id = Column( + Integer, + ForeignKey("process_tracker.dependency_type_lkup.dependency_type_id"), + nullable=False, + default=0, + ) child_extract = relationship("Extract", foreign_keys=[child_extract_id]) parent_extract = relationship("Extract", foreign_keys=[parent_extract_id]) diff --git a/process_tracker/models/process.py b/process_tracker/models/process.py index 84ae32f..54326fd 100755 --- a/process_tracker/models/process.py +++ b/process_tracker/models/process.py @@ -18,6 +18,26 @@ from process_tracker.models.model_base import default_date, Base, BaseColumn +class DependencyType(Base, BaseColumn): + + __tablename__ = "dependency_type_lkup" + __table_args__ = {"schema": "process_tracker"} + + dependency_type_id = Column( + Integer, + Sequence( + "dependency_type_lkup_dependency_type_id_seq", schema="process_tracker" + ), + primary_key=True, + nullable=False, + ) + dependency_type_name = Column(String(75), unique=True, nullable=False) + + def __repr__(self): + + return "" % self.dependency_type_name + + class ErrorType(Base, BaseColumn): __tablename__ = "error_type_lkup" @@ -473,6 +493,12 @@ class ProcessDependency(Base, BaseColumn): primary_key=True, nullable=False, ) + dependency_type_id = Column( + Integer, + ForeignKey("process_tracker.dependency_type_lkup.dependency_type_id"), + nullable=False, + default=0, + ) child_process = relationship("Process", foreign_keys=[child_process_id]) parent_process = relationship("Process", foreign_keys=[parent_process_id]) From 0bd6b52ca31707f71776c6bbe288b0df099326d2 Mon Sep 17 00:00:00 2001 From: Alex Meadows Date: Thu, 19 Dec 2019 10:12:36 -0500 Subject: [PATCH 2/5] process_tracker_python-144 Add Process Dependency Classifier :sparkles: Dependency type added Extract and Process dependencies can now be classified by type. --- dbscripts/mysql_process_tracker_defaults.sql | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dbscripts/mysql_process_tracker_defaults.sql b/dbscripts/mysql_process_tracker_defaults.sql index 0c0ad18..228c00f 100644 --- a/dbscripts/mysql_process_tracker_defaults.sql +++ b/dbscripts/mysql_process_tracker_defaults.sql @@ -55,3 +55,7 @@ INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (3, 'Internal'); INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (4, 'External'); +INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (0, 'Undefined'); +INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (1, 'Hard'); +INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (2, 'Soft'); + From 6a20306c6a6e75347928768f5419ee8ec01f1181 Mon Sep 17 00:00:00 2001 From: Alex Meadows Date: Thu, 19 Dec 2019 10:15:32 -0500 Subject: [PATCH 3/5] process_tracker_python-144 Add Process Dependency Classifier :sparkles: Dependency type added Extract and Process dependencies can now be classified by type. --- dbscripts/mysql_process_tracker_defaults.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbscripts/mysql_process_tracker_defaults.sql b/dbscripts/mysql_process_tracker_defaults.sql index 228c00f..7a0841e 100644 --- a/dbscripts/mysql_process_tracker_defaults.sql +++ b/dbscripts/mysql_process_tracker_defaults.sql @@ -56,6 +56,6 @@ INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (4, 'External'); INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (0, 'Undefined'); -INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (1, 'Hard'); -INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (2, 'Soft'); +INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (3, 'Hard'); +INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (4, 'Soft'); From ad46010c815744464c6fccb743cd0452277c7bd3 Mon Sep 17 00:00:00 2001 From: Alex Meadows Date: Thu, 19 Dec 2019 10:22:00 -0500 Subject: [PATCH 4/5] process_tracker_python-144 Add Process Dependency Classifier :sparkles: Dependency type added Extract and Process dependencies can now be classified by type. --- dbscripts/mysql_process_tracker.sql | 4 ++-- dbscripts/mysql_process_tracker_defaults.sql | 6 +++--- dbscripts/postgresql_process_tracker.sql | 4 ++-- dbscripts/postgresql_process_tracker_defaults.sql | 6 +++--- process_tracker/utilities/aws_utilities.py | 5 +++++ 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/dbscripts/mysql_process_tracker.sql b/dbscripts/mysql_process_tracker.sql index f2cf0c4..eb3abf7 100644 --- a/dbscripts/mysql_process_tracker.sql +++ b/dbscripts/mysql_process_tracker.sql @@ -266,7 +266,7 @@ create table process_tracker.extract_dependency ( parent_extract_id int not null, child_extract_id int not null, - dependency_type_id int default 0 not null, + dependency_type_id int default 1 not null, created_date_time timestamp default CURRENT_TIMESTAMP not null, created_by int default 0 not null, update_date_time timestamp default CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP not null, @@ -389,7 +389,7 @@ create table process_dependency ( parent_process_id int not null, child_process_id int not null, - dependency_type_id int default 0 not null, + dependency_type_id int default 1 not null, created_date_time timestamp default CURRENT_TIMESTAMP not null, created_by int default 0 not null, update_date_time timestamp default CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP not null, diff --git a/dbscripts/mysql_process_tracker_defaults.sql b/dbscripts/mysql_process_tracker_defaults.sql index 7a0841e..d78072f 100644 --- a/dbscripts/mysql_process_tracker_defaults.sql +++ b/dbscripts/mysql_process_tracker_defaults.sql @@ -55,7 +55,7 @@ INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (3, 'Internal'); INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (4, 'External'); -INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (0, 'Undefined'); -INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (3, 'Hard'); -INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (4, 'Soft'); +INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (1, 'Undefined'); +INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (2, 'Hard'); +INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (3, 'Soft'); diff --git a/dbscripts/postgresql_process_tracker.sql b/dbscripts/postgresql_process_tracker.sql index 1fccd09..ad22063 100644 --- a/dbscripts/postgresql_process_tracker.sql +++ b/dbscripts/postgresql_process_tracker.sql @@ -421,7 +421,7 @@ create table process_dependency references process, constraint process_dependency_pk primary key (child_process_id, parent_process_id), - dependency_type_id int default 0 not null + dependency_type_id int default 1 not null constraint process_dependency_fk03 references dependency_type_lkup, created_date_time timestamp with time zone default CURRENT_TIMESTAMP not null, @@ -1054,7 +1054,7 @@ create table extract_dependency child_extract_id integer not null constraint extract_dependency_fk02 references extract_tracking, - dependency_type_id int default 0 not null + dependency_type_id int default 1 not null constraint extract_dependency_fk03 references dependency_type_lkup, constraint extract_dependency_pk diff --git a/dbscripts/postgresql_process_tracker_defaults.sql b/dbscripts/postgresql_process_tracker_defaults.sql index 26a62f3..84fdf6c 100644 --- a/dbscripts/postgresql_process_tracker_defaults.sql +++ b/dbscripts/postgresql_process_tracker_defaults.sql @@ -54,6 +54,6 @@ INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (3, 'Internal'); INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (4, 'External'); -INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (0, 'Undefined'); -INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (1, 'Hard'); -INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (2, 'Soft'); \ No newline at end of file +INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (1, 'Undefined'); +INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (2, 'Hard'); +INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (3, 'Soft'); \ No newline at end of file diff --git a/process_tracker/utilities/aws_utilities.py b/process_tracker/utilities/aws_utilities.py index f375f35..ff79b17 100644 --- a/process_tracker/utilities/aws_utilities.py +++ b/process_tracker/utilities/aws_utilities.py @@ -15,6 +15,11 @@ def __init__(self): self.logger = logging.getLogger(__name__) self.logger.setLevel(self.log_level) + logging.getLogger("boto3").setLevel(logging.CRITICAL) + logging.getLogger("botocore").setLevel(logging.CRITICAL) + logging.getLogger("s3transfer").setLevel(logging.CRITICAL) + logging.getLogger("urllib3").setLevel(logging.CRITICAL) + self.s3 = boto3.resource("s3") self.url_match = re.compile( From 6061fff1107b3542843301a971241bb8556f0952 Mon Sep 17 00:00:00 2001 From: Alex Meadows Date: Thu, 19 Dec 2019 10:25:26 -0500 Subject: [PATCH 5/5] process_tracker_python-144 Add Process Dependency Classifier :sparkles: Dependency type added Extract and Process dependencies can now be classified by type. --- process_tracker/models/extract.py | 2 +- process_tracker/models/process.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/process_tracker/models/extract.py b/process_tracker/models/extract.py index 169ae19..fe38c58 100755 --- a/process_tracker/models/extract.py +++ b/process_tracker/models/extract.py @@ -215,7 +215,7 @@ class ExtractDependency(Base, BaseColumn): Integer, ForeignKey("process_tracker.dependency_type_lkup.dependency_type_id"), nullable=False, - default=0, + default=1, ) child_extract = relationship("Extract", foreign_keys=[child_extract_id]) diff --git a/process_tracker/models/process.py b/process_tracker/models/process.py index 54326fd..a35d3b3 100755 --- a/process_tracker/models/process.py +++ b/process_tracker/models/process.py @@ -497,7 +497,7 @@ class ProcessDependency(Base, BaseColumn): Integer, ForeignKey("process_tracker.dependency_type_lkup.dependency_type_id"), nullable=False, - default=0, + default=1, ) child_process = relationship("Process", foreign_keys=[child_process_id])