diff --git a/dbscripts/mysql_process_tracker.sql b/dbscripts/mysql_process_tracker.sql index d353f11..eb3abf7 100644 --- a/dbscripts/mysql_process_tracker.sql +++ b/dbscripts/mysql_process_tracker.sql @@ -1,5 +1,18 @@ USE process_tracker; +create table process_tracker.dependency_type_lkup +( + dependency_type_id int auto_increment + primary key, + dependency_type_name varchar(75) not null, + created_date_time timestamp default CURRENT_TIMESTAMP not null, + created_by int default 0 not null, + update_date_time timestamp default CURRENT_TIMESTAMP not null on update CURRENT_TIMESTAMP, + updated_by int default 0 not null, + constraint dependency_type_lkup_dependency_type_name_uindex + unique (dependency_type_name) +); + create table process_tracker.character_set_lkup ( character_set_id int auto_increment @@ -253,6 +266,7 @@ create table process_tracker.extract_dependency ( parent_extract_id int not null, child_extract_id int not null, + dependency_type_id int default 1 not null, created_date_time timestamp default CURRENT_TIMESTAMP not null, created_by int default 0 not null, update_date_time timestamp default CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP not null, @@ -261,7 +275,9 @@ create table process_tracker.extract_dependency constraint extract_dependency_fk01 foreign key (parent_extract_id) references extract_tracking (extract_id), constraint extract_dependency_fk02 - foreign key (child_extract_id) references extract_tracking (extract_id) + foreign key (child_extract_id) references extract_tracking (extract_id), + constraint extract_dependency_fk03 + foreign key (dependency_type_id) references dependency_type_lkup (dependency_type_id) ) comment 'Table tracking interdependencies between extract files.'; @@ -373,6 +389,7 @@ create table process_dependency ( parent_process_id int not null, child_process_id int not null, + dependency_type_id int default 1 not null, created_date_time timestamp default CURRENT_TIMESTAMP not null, created_by int default 0 not null, update_date_time timestamp default CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP not null, @@ -381,7 +398,9 @@ create table process_dependency constraint process_dependency_ibfk_1 foreign key (parent_process_id) references process (process_id), constraint process_dependency_ibfk_2 - foreign key (child_process_id) references process (process_id) + foreign key (child_process_id) references process (process_id), + constraint process_dependency_fk03 + foreign key (dependency_type_id) references dependency_type_lkup (dependency_type_id) ); create index child_process_id diff --git a/dbscripts/mysql_process_tracker_defaults.sql b/dbscripts/mysql_process_tracker_defaults.sql index 9e5e927..d78072f 100644 --- a/dbscripts/mysql_process_tracker_defaults.sql +++ b/dbscripts/mysql_process_tracker_defaults.sql @@ -53,4 +53,9 @@ INSERT INTO process_tracker.filesize_type_lkup (filesize_type_id, filesize_type_ INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (1, 'Undefined'); INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (2, 'Database'); INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (3, 'Internal'); -INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (4, 'External'); \ No newline at end of file +INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (4, 'External'); + +INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (1, 'Undefined'); +INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (2, 'Hard'); +INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (3, 'Soft'); + diff --git a/dbscripts/postgresql_process_tracker.sql b/dbscripts/postgresql_process_tracker.sql index c004e97..ad22063 100644 --- a/dbscripts/postgresql_process_tracker.sql +++ b/dbscripts/postgresql_process_tracker.sql @@ -7,6 +7,21 @@ create schema process_tracker; alter schema process_tracker owner to pt_admin; +create table process_tracker.dependency_type_lkup +( + dependency_type_id serial not null + constraint dependency_type_lkup_pk + primary key, + dependency_type_name varchar(75) not null, + created_date_time timestamptz default CURRENT_TIMESTAMP not null, + created_by int default 0 not null, + update_date_time timestamptz default CURRENT_TIMESTAMP not null, + updated_by int default 0 not null +); + +create unique index dependency_type_lkup_dependency_type_name_uindex + on process_tracker.dependency_type_lkup (dependency_type_name); + create table dataset_type_lkup ( dataset_type_id serial not null @@ -406,6 +421,9 @@ create table process_dependency references process, constraint process_dependency_pk primary key (child_process_id, parent_process_id), + dependency_type_id int default 1 not null + constraint process_dependency_fk03 + references dependency_type_lkup, created_date_time timestamp with time zone default CURRENT_TIMESTAMP not null, created_by integer default 0 not null, update_date_time timestamp with time zone default CURRENT_TIMESTAMP not null, @@ -1036,6 +1054,9 @@ create table extract_dependency child_extract_id integer not null constraint extract_dependency_fk02 references extract_tracking, + dependency_type_id int default 1 not null + constraint extract_dependency_fk03 + references dependency_type_lkup, constraint extract_dependency_pk primary key (parent_extract_id, child_extract_id), created_date_time timestamp with time zone default CURRENT_TIMESTAMP not null, @@ -1136,6 +1157,8 @@ CREATE TRIGGER data_type_lkup_update_date_time_trg BEFORE UPDATE ON process_tracker.data_type_lkup FOR EACH ROW EXECUTE PROCEDURE update_date_time_trigger(); CREATE TRIGGER dataset_type_lkup_update_date_time_trg BEFORE UPDATE ON process_tracker.dataset_type_lkup FOR EACH ROW EXECUTE PROCEDURE update_date_time_trigger(); +CREATE TRIGGER dependency_type_lkup_update_date_time_trg BEFORE UPDATE + ON process_tracker.dependency_type_lkup FOR EACH ROW EXECUTE PROCEDURE update_date_time_trigger(); CREATE TRIGGER error_tracking_update_date_time_trg BEFORE UPDATE ON process_tracker.error_tracking FOR EACH ROW EXECUTE PROCEDURE update_date_time_trigger(); CREATE TRIGGER error_type_lkup_update_date_time_trg BEFORE UPDATE diff --git a/dbscripts/postgresql_process_tracker_defaults.sql b/dbscripts/postgresql_process_tracker_defaults.sql index cf2efde..84fdf6c 100644 --- a/dbscripts/postgresql_process_tracker_defaults.sql +++ b/dbscripts/postgresql_process_tracker_defaults.sql @@ -52,4 +52,8 @@ INSERT INTO process_tracker.filesize_type_lkup (filesize_type_id, filesize_type_ INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (1, 'Undefined'); INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (2, 'Database'); INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (3, 'Internal'); -INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (4, 'External'); \ No newline at end of file +INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (4, 'External'); + +INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (1, 'Undefined'); +INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (2, 'Hard'); +INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (3, 'Soft'); \ No newline at end of file diff --git a/process_tracker/models/extract.py b/process_tracker/models/extract.py index ae9f69a..fe38c58 100755 --- a/process_tracker/models/extract.py +++ b/process_tracker/models/extract.py @@ -211,6 +211,12 @@ class ExtractDependency(Base, BaseColumn): primary_key=True, nullable=False, ) + dependency_type_id = Column( + Integer, + ForeignKey("process_tracker.dependency_type_lkup.dependency_type_id"), + nullable=False, + default=1, + ) child_extract = relationship("Extract", foreign_keys=[child_extract_id]) parent_extract = relationship("Extract", foreign_keys=[parent_extract_id]) diff --git a/process_tracker/models/process.py b/process_tracker/models/process.py index 84ae32f..a35d3b3 100755 --- a/process_tracker/models/process.py +++ b/process_tracker/models/process.py @@ -18,6 +18,26 @@ from process_tracker.models.model_base import default_date, Base, BaseColumn +class DependencyType(Base, BaseColumn): + + __tablename__ = "dependency_type_lkup" + __table_args__ = {"schema": "process_tracker"} + + dependency_type_id = Column( + Integer, + Sequence( + "dependency_type_lkup_dependency_type_id_seq", schema="process_tracker" + ), + primary_key=True, + nullable=False, + ) + dependency_type_name = Column(String(75), unique=True, nullable=False) + + def __repr__(self): + + return "" % self.dependency_type_name + + class ErrorType(Base, BaseColumn): __tablename__ = "error_type_lkup" @@ -473,6 +493,12 @@ class ProcessDependency(Base, BaseColumn): primary_key=True, nullable=False, ) + dependency_type_id = Column( + Integer, + ForeignKey("process_tracker.dependency_type_lkup.dependency_type_id"), + nullable=False, + default=1, + ) child_process = relationship("Process", foreign_keys=[child_process_id]) parent_process = relationship("Process", foreign_keys=[parent_process_id]) diff --git a/process_tracker/utilities/aws_utilities.py b/process_tracker/utilities/aws_utilities.py index f375f35..ff79b17 100644 --- a/process_tracker/utilities/aws_utilities.py +++ b/process_tracker/utilities/aws_utilities.py @@ -15,6 +15,11 @@ def __init__(self): self.logger = logging.getLogger(__name__) self.logger.setLevel(self.log_level) + logging.getLogger("boto3").setLevel(logging.CRITICAL) + logging.getLogger("botocore").setLevel(logging.CRITICAL) + logging.getLogger("s3transfer").setLevel(logging.CRITICAL) + logging.getLogger("urllib3").setLevel(logging.CRITICAL) + self.s3 = boto3.resource("s3") self.url_match = re.compile(