file movement fixed to match recurring tasks in X3.

readme.txt

@@ -19,4 +19,7 @@ step 1 read in file
 
 ----------------------------------------------------------------------------------------
 
+TODO: schedule X3 batch server to process orders and then deliveries, but when do we receive files, how often should this be happening?
+see ZECMSL recurring task
+
 

source_ecommerce_make_order.py

@@ -1,4 +1,12 @@
 #!/usr/bin/env python3
+"""
+Pull shipment files from the Stash AWS FTP.
+After copying file, move file to archive folder on FTP.
+Create ZSHPORD import files, which X3 will consume on a schedule via ZECMSL recurring task.
+Copy file to the archive and then pass it to the shipment maker directory.
+
+TODO: Source needs to send us real data for final adjustments (site, discount, multilot, etc.)
+"""
 import csv
 import pprint
 import dataclasses
@@ -9,27 +17,34 @@ import pathlib
 import re
 import shutil
 import typing
+import paramiko
 
 import records  # type: ignore
 
 import yamamotoyama  # type: ignore
 import yamamotoyama.x3_imports  # type: ignore
 
+THIS_DIRECTORY = pathlib.Path(__file__).parent
+
 SFTP_HOST = "s-8ade4d252cc44c50b.server.transfer.us-west-1.amazonaws.com"
 SFTP_USERNAME = "yumiddleware2023"
-SSH_DIRECTORY = edi_940.THIS_DIRECTORY / "ssh" #TODO fixme
+SSH_DIRECTORY = THIS_DIRECTORY / "ssh" #TODO fixme
 SSH_KNOWN_HOSTS_FILE = str(SSH_DIRECTORY / "known_hosts")
 SSH_KEY_FILENAME = str(SSH_DIRECTORY / "id_ed25519")
 
-THIS_DIRECTORY = pathlib.Path(__file__).parent
 INCOMING_DIRECTORY = THIS_DIRECTORY / "incoming_orders"
+SHIPMENTS_DIRECTORY = THIS_DIRECTORY / "incoming_shipments"
 SOH_IMPORT_DIRECTORY = THIS_DIRECTORY / "to_import_SOH"
 
 def main():
-    retrieve_x12_edi_files()
+    #retrieve_x12_edi_files()#TODO uncomment
     for file in INCOMING_DIRECTORY.iterdir():
-        process_files()
+        if file.name[-4:] != '.csv':
-        shutil.move(file, INCOMING_DIRECTORY / "archive" / file.name)
+            continue
+        else:
+            process_files(file)
+            shutil.move(file, SHIPMENTS_DIRECTORY / file.name)
+            #shutil.move(file, INCOMING_DIRECTORY / "archive" / file.name)#archives go in the shipping folder
             combine_zshpords()
         #TODO determine X3 processing schedule
 
@@ -48,11 +63,18 @@ def retrieve_x12_edi_files():
     """
     Connect to S3 bucket & pull down files.
     """
-    with sftp_server() as sftp_connection:
+    with paramiko.SSHClient() as ssh_client:
+        ssh_client.load_system_host_keys()
+        ssh_client.load_host_keys(SSH_KNOWN_HOSTS_FILE)
+        ssh_client.set_missing_host_key_policy(paramiko.client.RejectPolicy)
+        ssh_client.connect(
+            hostname=SFTP_HOST, username=SFTP_USERNAME, key_filename=SSH_KEY_FILENAME
+        )
+        with ssh_client.open_sftp() as sftp_connection:
             sftp_connection.chdir("/yu-edi-transfer/source-logi/dev/ecomm-inbound")#TODO set to prod
             for filename in sftp_connection.listdir():
-            if edi_945.SOURCE_945_FILENAME_RE.match(filename):
+                #if edi_945.SOURCE_945_FILENAME_RE.match(filename):#TODO fixme
-                sftp_connection.get(filename, edi_945.X12_DIRECTORY / filename)
+                sftp_connection.get(filename, INCOMING_DIRECTORY / filename)
                 new_filename = f"/yu-edi-transfer/source-logi/dev/ecomm-processed/{filename}"#TODO set to prod
                 sftp_connection.rename(filename, new_filename)
 
@@ -60,12 +82,12 @@ def combine_zshpords():
     """
     Collect all ZSHPORD imports into a single file for easy import.
     """
-    archive_directory = INCOMING_DIRECTORY / "archive"
+    archive_directory = SOH_IMPORT_DIRECTORY / "archive"
     archive_directory.mkdir(exist_ok=True)
-    with (INCOMING_DIRECTORY / "ZSHPORD.dat").open(
+    with (SOH_IMPORT_DIRECTORY / "ZSHPORD.dat").open(
         "a", encoding="utf-8", newline="\n"
     ) as combined_import_file:
-        for individual_import_filename in INCOMING_DIRECTORY.glob(
+        for individual_import_filename in SOH_IMPORT_DIRECTORY.glob(
             "ZSHPORD_*.dat"
         ):
             with individual_import_filename.open(
@@ -106,8 +128,8 @@ def process_files(file):
                 sales_order.header.stofcy = ship_site
                 sales_order.header.bpdnam = customer_name
                 sales_order.header.invdtaamt_5 = ship_charge
-                sales_order.header.invdtaamt_7 = '0.33' #discount
+                sales_order.header.invdtaamt_7 = '0.33' #TODO discount
-                sales_order.header.invdtaamt_8 = '0.51'#taxes
+                sales_order.header.invdtaamt_8 = '0.51'#TODO taxes
             
             #gather line data
             line_product = row[0]
@@ -123,7 +145,7 @@ def process_files(file):
             )
         time_stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
         with yamamotoyama.x3_imports.open_import_file(
-            SOH_IMPORT_DIRECTORY / f"ZSHPORD_{time_stamp}.dat"
+            SOH_IMPORT_DIRECTORY / f"ZSHPORD_{file.name}_{time_stamp}.dat"
         ) as import_file:
             sales_order.output(import_file)
 

source_ecommerce_make_shipment.py

@@ -20,9 +20,37 @@ INCOMING_DIRECTORY = THIS_DIRECTORY / "incoming_shipments"
 SDH_IMPORT_DIRECTORY = THIS_DIRECTORY / "to_import_SDH"
 
 def main():
-    process_files()
+    for file in INCOMING_DIRECTORY.iterdir():
-    #TODO shutil the files around, archive on the ftp
+        if file.name[-4:] != '.csv':
+            continue
+        else:
+            process_files(file)
+            shutil.move(file, INCOMING_DIRECTORY / "archive" / file.name)
+            combine_zship945s()
         #TODO determine X3 processing schedule
+        #TODO determine X3 processing schedule
+
+def combine_zship945s():
+    """
+    Collect all ZSHPORD imports into a single file for easy import.
+    """
+    archive_directory = SDH_IMPORT_DIRECTORY / "archive"
+    archive_directory.mkdir(exist_ok=True)
+    with (SDH_IMPORT_DIRECTORY / "ZSHIP945S.dat").open(
+        "a", encoding="utf-8", newline="\n"
+    ) as combined_import_file:
+        for individual_import_filename in SDH_IMPORT_DIRECTORY.glob(
+            "ZSHIP945S_*.dat"
+        ):
+            with individual_import_filename.open(
+                "r", encoding="utf-8", newline="\n"
+            ) as individual_import_file:
+                for line in individual_import_file:
+                    combined_import_file.write(line)
+            shutil.move(
+                individual_import_filename,
+                archive_directory / individual_import_filename.name,
+            )
 
 def find_so_from_po(cust_po):
     with yamamotoyama.get_connection('test') as db_connection:#TODO remove 'test'
@@ -39,8 +67,7 @@ def find_so_from_po(cust_po):
             order=cust_po,
         ).first()["SOHNUM_0"]
 
-def process_files():
+def process_files(file):
-    for file in INCOMING_DIRECTORY.iterdir():
     with open(file) as source_file:
         csv_reader = csv.reader(source_file)
         warehouse_shipment = WarehouseShipment()