Process tracker python 144

dbscripts/mysql_process_tracker.sql

@@ -1,5 +1,18 @@
 USE process_tracker;
 
+create table process_tracker.dependency_type_lkup
+(
+	dependency_type_id int auto_increment
+		primary key,
+	dependency_type_name varchar(75) not null,
+	created_date_time timestamp default CURRENT_TIMESTAMP not null,
+	created_by int default 0 not null,
+	update_date_time timestamp default CURRENT_TIMESTAMP not null on update CURRENT_TIMESTAMP,
+	updated_by int default 0 not null,
+	constraint dependency_type_lkup_dependency_type_name_uindex
+		unique (dependency_type_name)
+);
+
 create table process_tracker.character_set_lkup
 (
 	character_set_id int auto_increment
@@ -253,6 +266,7 @@ create table process_tracker.extract_dependency
 (
 	parent_extract_id int not null,
 	child_extract_id int not null,
+	dependency_type_id int default 1 not null,
 	created_date_time timestamp default CURRENT_TIMESTAMP not null,
 	created_by int default 0 not null,
 	update_date_time timestamp default CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP not null,
@@ -261,7 +275,9 @@ create table process_tracker.extract_dependency
 	constraint extract_dependency_fk01
 		foreign key (parent_extract_id) references extract_tracking (extract_id),
 	constraint extract_dependency_fk02
-		foreign key (child_extract_id) references extract_tracking (extract_id)
+		foreign key (child_extract_id) references extract_tracking (extract_id),
+	constraint extract_dependency_fk03
+		foreign key (dependency_type_id) references dependency_type_lkup (dependency_type_id)
 )
 comment 'Table tracking interdependencies between extract files.';
 
@@ -373,6 +389,7 @@ create table process_dependency
 (
 	parent_process_id int not null,
 	child_process_id int not null,
+	dependency_type_id int default 1 not null,
 	created_date_time timestamp default CURRENT_TIMESTAMP not null,
 	created_by int default 0 not null,
 	update_date_time timestamp default CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP not null,
@@ -381,7 +398,9 @@ create table process_dependency
 	constraint process_dependency_ibfk_1
 		foreign key (parent_process_id) references process (process_id),
 	constraint process_dependency_ibfk_2
-		foreign key (child_process_id) references process (process_id)
+		foreign key (child_process_id) references process (process_id),
+	constraint process_dependency_fk03
+		foreign key (dependency_type_id) references dependency_type_lkup (dependency_type_id)
 );
 
 create index child_process_id

dbscripts/mysql_process_tracker_defaults.sql

@@ -53,4 +53,9 @@ INSERT INTO process_tracker.filesize_type_lkup (filesize_type_id, filesize_type_
 INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (1, 'Undefined');
 INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (2, 'Database');
 INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (3, 'Internal');
-INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (4, 'External');
+INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (4, 'External');
+
+INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (1, 'Undefined');
+INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (2, 'Hard');
+INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (3, 'Soft');
+

dbscripts/postgresql_process_tracker.sql

@@ -7,6 +7,21 @@ create schema process_tracker;
 
 alter schema process_tracker owner to pt_admin;
 
+create table process_tracker.dependency_type_lkup
+(
+	dependency_type_id serial not null
+		constraint dependency_type_lkup_pk
+			primary key,
+	dependency_type_name varchar(75) not null,
+	created_date_time timestamptz default CURRENT_TIMESTAMP not null,
+	created_by int default 0 not null,
+	update_date_time timestamptz default CURRENT_TIMESTAMP not null,
+	updated_by int default 0 not null
+);
+
+create unique index dependency_type_lkup_dependency_type_name_uindex
+	on process_tracker.dependency_type_lkup (dependency_type_name);
+
 create table dataset_type_lkup
 (
 	dataset_type_id serial not null
@@ -406,6 +421,9 @@ create table process_dependency
 			references process,
 	constraint process_dependency_pk
 		primary key (child_process_id, parent_process_id),
+	dependency_type_id int default 1 not null
+	    constraint process_dependency_fk03
+		    references dependency_type_lkup,
 	created_date_time timestamp with time zone default CURRENT_TIMESTAMP not null,
 	created_by integer default 0 not null,
 	update_date_time timestamp with time zone default CURRENT_TIMESTAMP not null,
@@ -1036,6 +1054,9 @@ create table extract_dependency
 	child_extract_id integer not null
 		constraint extract_dependency_fk02
 			references extract_tracking,
+	dependency_type_id int default 1 not null
+	    constraint extract_dependency_fk03
+		    references dependency_type_lkup,
 	constraint extract_dependency_pk
 		primary key (parent_extract_id, child_extract_id),
 	created_date_time timestamp with time zone default CURRENT_TIMESTAMP not null,
@@ -1136,6 +1157,8 @@ CREATE TRIGGER data_type_lkup_update_date_time_trg BEFORE UPDATE
     ON process_tracker.data_type_lkup FOR EACH ROW EXECUTE PROCEDURE update_date_time_trigger();
 CREATE TRIGGER dataset_type_lkup_update_date_time_trg BEFORE UPDATE
     ON process_tracker.dataset_type_lkup FOR EACH ROW EXECUTE PROCEDURE update_date_time_trigger();
+CREATE TRIGGER dependency_type_lkup_update_date_time_trg BEFORE UPDATE
+    ON process_tracker.dependency_type_lkup FOR EACH ROW EXECUTE PROCEDURE update_date_time_trigger();
 CREATE TRIGGER error_tracking_update_date_time_trg BEFORE UPDATE
     ON process_tracker.error_tracking FOR EACH ROW EXECUTE PROCEDURE update_date_time_trigger();
 CREATE TRIGGER error_type_lkup_update_date_time_trg BEFORE UPDATE

dbscripts/postgresql_process_tracker_defaults.sql

@@ -52,4 +52,8 @@ INSERT INTO process_tracker.filesize_type_lkup (filesize_type_id, filesize_type_
 INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (1, 'Undefined');
 INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (2, 'Database');
 INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (3, 'Internal');
-INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (4, 'External');
+INSERT INTO process_tracker.source_type_lkup (source_type_id, source_type_name) VALUES (4, 'External');
+
+INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (1, 'Undefined');
+INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (2, 'Hard');
+INSERT INTO process_tracker.dependency_type_lkup (dependency_type_id, dependency_type_name) VALUES (3, 'Soft');

process_tracker/models/extract.py

@@ -211,6 +211,12 @@ class ExtractDependency(Base, BaseColumn):
         primary_key=True,
         nullable=False,
     )
+    dependency_type_id = Column(
+        Integer,
+        ForeignKey("process_tracker.dependency_type_lkup.dependency_type_id"),
+        nullable=False,
+        default=1,
+    )
 
     child_extract = relationship("Extract", foreign_keys=[child_extract_id])
     parent_extract = relationship("Extract", foreign_keys=[parent_extract_id])

process_tracker/models/process.py

@@ -18,6 +18,26 @@
 from process_tracker.models.model_base import default_date, Base, BaseColumn
 
 
+class DependencyType(Base, BaseColumn):
+
+    __tablename__ = "dependency_type_lkup"
+    __table_args__ = {"schema": "process_tracker"}
+
+    dependency_type_id = Column(
+        Integer,
+        Sequence(
+            "dependency_type_lkup_dependency_type_id_seq", schema="process_tracker"
+        ),
+        primary_key=True,
+        nullable=False,
+    )
+    dependency_type_name = Column(String(75), unique=True, nullable=False)
+
+    def __repr__(self):
+
+        return "<DependencyType (name=%s)>" % self.dependency_type_name
+
+
 class ErrorType(Base, BaseColumn):
 
     __tablename__ = "error_type_lkup"
@@ -473,6 +493,12 @@ class ProcessDependency(Base, BaseColumn):
         primary_key=True,
         nullable=False,
     )
+    dependency_type_id = Column(
+        Integer,
+        ForeignKey("process_tracker.dependency_type_lkup.dependency_type_id"),
+        nullable=False,
+        default=1,
+    )
 
     child_process = relationship("Process", foreign_keys=[child_process_id])
     parent_process = relationship("Process", foreign_keys=[parent_process_id])

process_tracker/utilities/aws_utilities.py

@@ -15,6 +15,11 @@ def __init__(self):
         self.logger = logging.getLogger(__name__)
         self.logger.setLevel(self.log_level)
 
+        logging.getLogger("boto3").setLevel(logging.CRITICAL)
+        logging.getLogger("botocore").setLevel(logging.CRITICAL)
+        logging.getLogger("s3transfer").setLevel(logging.CRITICAL)
+        logging.getLogger("urllib3").setLevel(logging.CRITICAL)
+
         self.s3 = boto3.resource("s3")
 
         self.url_match = re.compile(