Reworked 867 process and minor updates to others.

edi_846.py

@@ -49,7 +49,7 @@ def main():
         if SHANDEX_846_FILENAME_RE.match(edi_filename.name):
             shandex_inventory=process_file(edi_filename)
             # file moved to 997 processing folder to be sent later
-            #shutil.move(edi_filename, EDI_997_DIRECTORY / edi_filename.name)
+            shutil.move(edi_filename, EDI_997_DIRECTORY / edi_filename.name)
             #get stock information about WON and store it
             #pass date from EDI so we can subtract newer stock movements?
             x3_won_inventory=get_x3_won_inventory()
@@ -214,7 +214,8 @@ def stock_count_alert():
     msg['Subject'] = 'New Stock Count from Shandex'
     msg['Precedence'] = 'bulk'
     msg['From'] = 'x3report@stashtea.com'
-    msg['To'] = 'bleeson@stashtea.com'#TODO correct receipientscares
+    msg['To'] = 'isenn@yamamotoyama.com,vgomez@yamamotoyama.com'
+    msg['Cc'] = 'bleeson@stashtea.com'
     emailtext = f'Attached.'
     msg.attach(MIMEText(emailtext, 'plain'))
     for file in EDI_846_ATTACHMENTS.iterdir():

edi_943.py

@@ -19,7 +19,7 @@ import yamamotoyama  # type:ignore
 SITE_MAPPING = {
 'WNJ' : 'SOURCELOGISTICS',
 'WCA' : 'SOURCELOGISTICS',
-'WON' : 'SHANDEXTEST    ' # TODO CHANGE TO SHANDEX, needs to be 15 characters
+'WON' : 'SHANDEX        ' #Testing value is SHANDEXTEST with spaces to 15 characters
 }
 
 SHIPPING_CODE_MAPPING = {
@@ -35,14 +35,12 @@ SHIPPING_CODE_MAPPING = {
 THIS_DIRECTORY = pathlib.Path(__file__).parent
 X12_SHANDEX = THIS_DIRECTORY / "outgoing"
 
-# X12_SOURCELOGISTICS = THIS_DIRECTORY / "edi-testing" #test directories
-# X12_SHANDEX = THIS_DIRECTORY / "edi-testing"
 
 def main():
     """
     Do it!
     """
-    with yamamotoyama.get_connection('test') as database:
+    with yamamotoyama.get_connection() as database:
         shipments = list(get_shipments(database))
         for shipment in shipments:
             write_943(database, shipment)
@@ -127,7 +125,7 @@ def write_943(database: records.Connection, shipment: str):
     with database.transaction() as _:
         database.query(
             """
-            update [FY23TEST].[SDELIVERY]
+            update [PROD].[SDELIVERY]
             set [XX4S_943RDY_0] = 1
             where [SOHNUM_0] = :shipment
             """,
@@ -136,7 +134,7 @@ def write_943(database: records.Connection, shipment: str):
         order = get_order_for_shipment(database, shipment)
         database.query(
             """
-            update [FY23TEST].[SORDER]
+            update [PROD].[SORDER]
             set [XX4S_UDF2_0] = :sent_message
             where [SOHNUM_0] = :order
             """,
@@ -153,7 +151,7 @@ def get_shipment_destination(database: records.Connection, shipment: str) -> str
             """
         select
             [SDH].[BPCORD_0]
-        from [FY23TEST].[SDELIVERY] as [SDH]
+        from [PROD].[SDELIVERY] as [SDH]
         where
             [SDH].[SDHNUM_0] = :shipment
         """,
@@ -172,7 +170,7 @@ def get_order_for_shipment(database: records.Connection, shipment: str) -> str:
             """
         select
             [SDH].[SOHNUM_0]
-        from [FY23TEST].[SDELIVERY] as [SDH]
+        from [PROD].[SDELIVERY] as [SDH]
         where
             [SDH].[SDHNUM_0] = :shipment
         """,
@@ -191,8 +189,8 @@ def get_shipments(database: records.Connection) -> typing.Iterator[str]:
         """
         select
             [SDH].[SDHNUM_0]
-        from [FY23TEST].[SDELIVERY] as [SDH]
+        from [PROD].[SDELIVERY] as [SDH]
-        join [FY23TEST].[SORDER] as [SOH]
+        join [PROD].[SORDER] as [SOH]
             on [SOH].[SOHNUM_0] = [SDH].[SOHNUM_0]
         where
             (
@@ -203,7 +201,7 @@ def get_shipments(database: records.Connection) -> typing.Iterator[str]:
             )
             and [SDH].[SHIDAT_0] >= {d'2023-10-09'}
             and (
-                [SOH].[XX4S_UDF2_0] = ''
+                [SOH].[XX4S_UDF2_0] not like '943%'
                 or [SDH].[XX4S_943RDY_0] = 2
             )
             and (
@@ -267,7 +265,7 @@ def get_shipment(
 		  ,[GROWEI_0]
 		  ,[CFMFLG_0]
 		  ,[SHIDAT_0]
-        from [FY23TEST].[zyumiddleware_shipment] as [SDH]
+        from [PROD].[zyumiddleware_shipment_shandex] as [SDH]
         where
             [SDH].[SDHNUM_0] = :shipment
         """,
@@ -549,7 +547,7 @@ class ShipmentDetail(X12):
                         "",
                         "",
                         self.gtin_or_upc_marker,
-                        self.gtin_or_upc_code,#W04-15
+                        self.gtin_or_upc_code.replace(' ',''),#W04-15
                     ]
                 ),
                 self.line(

edi_947.py

@@ -34,10 +34,6 @@ X12_DIRECTORY = THIS_DIRECTORY / "incoming"
 IMPORTS_DIRECTORY = THIS_DIRECTORY / "x3_imports"
 EDI_997_DIRECTORY = THIS_DIRECTORY / "997_processing"
 
-SHANDEX_947_FILENAME_RE = re.compile(
-    r"\A 947_YAMAMOTOYAMA_ \S+ [.]edi \Z", re.X | re.M | re.S
-)
-#TODO remove this and have a single production name? why is this one different
 SHANDEX_947_FILENAME_RE = re.compile(
     r"\A 947_QTY_STASH-YAMAMOTOYAMA_ \S+ [.]edi \Z", re.X | re.M | re.S
 )
@@ -80,7 +76,7 @@ def main():
         if SHANDEX_947_FILENAME_RE.match(edi_filename.name):
             process_file(edi_filename)
             # file moved to 997 processing folder to be sent later
-            #shutil.move(edi_filename, EDI_997_DIRECTORY / edi_filename.name)
+            shutil.move(edi_filename, EDI_997_DIRECTORY / edi_filename.name)
     combine_zscs()
 
 
@@ -154,7 +150,8 @@ def stock_movement_alert(itmref, qty, lot, status):
     msg['Subject'] = 'New Stock Change from Shandex'
     msg['Precedence'] = 'bulk'
     msg['From'] = 'x3report@stashtea.com'
-    msg['To'] = 'bleeson@stashtea.com'#TODO correct receipientscares
+    msg['To'] = 'isenn@yamamotoyama.com, vgomez@yamamotoyama.com'
+    msg['CC'] = 'bleeson@stashtea.com'
     emailtext = f'Item: {itmref}\nQty: {qty}\nLot: {lot}\nStatus: {DAMAGE_CODE_MAPPING[status]}\nReason: {DAMAGE_CODE_DESCRIPTIONS_MAPPING[status]}'
     msg.attach(MIMEText(emailtext, 'plain'))
     with smtplib.SMTP_SSL("smtp.gmail.com", 465) as smtp:
@@ -222,37 +219,6 @@ class StockChangeSubDetail:
     loc: str = ""
     sta: str = "A"
     
-    # def stojous(self, shipment, item) -> typing.List[str]:
-        # """
-        # Convert grouped lot quantities into individual STOJOU records to fit on stockchange
-        # """
-        # with yamamotoyama.get_connection('test') as database: #todo remove 'test'
-            # details = (
-                # database.query(
-                    # """
-                # select
-                    # 'S',
-                    # 'A',
-                    # [STJ].[PCU_0],
-                    # cast(cast(-1*[STJ].[QTYSTU_0] as int) as nvarchar),
-                    # [STJ].[LOT_0],
-                    # '',
-                    # '' 
-                # from [FY23TEST].[STOJOU] [STJ] --TODO change to PROD
-                # where 
-                    # [STJ].[VCRNUM_0] = :sdhnum
-                    # and [STJ].[ITMREF_0] = :itmref
-                    # and [STJ].[LOT_0] = :lot
-                    # and [STJ].[TRSTYP_0] = 4
-                # """,
-                    # sdhnum=shipment,
-                    # itmref=item,
-                    # lot=self.lot,
-                # )
-                # .all()
-            # )
-        # return details
-        
     
     def convert_to_strings(self) -> typing.List[str]:
         """
@@ -288,7 +254,7 @@ class StockChangeDetail:
     itmref: str = ""
     pcu: str = ""
     pcustucoe: int = 1 #does this need a lookup?
-    sta: str = "A" #todo this needs to flip based on the transaction A > R, A > Q, what about Q > A?
+    sta: str = "A"
     loctyp: str = ""
     loc: str = ""
     lot: str = ""
@@ -564,39 +530,6 @@ class StockChange:
             if value:
                 self._fill_info_from_shipment()
 
-    # def _get_shipment_from_x3(self) -> records.Record:
-        # """
-        # Fetch shipment from X3 database.
-        # """
-        # with yamamotoyama.get_connection('test') as db_connection:#todo remove 'test'
-            # return db_connection.query(
-                # """
-                # select
-                    # [SDH].[STOFCY_0],
-                    # [SDH].[SDHNUM_0],
-                    # [SDH].[SALFCY_0],
-                    # [SDH].[BPCORD_0],
-                    # [SDH].[CUR_0],
-                    # [SDH].[SOHNUM_0]
-                # from [FY23TEST].[SDELIVERY] [SDH]--TODO change back to [PROD]
-                # where 
-                    # [SDH].[SDHNUM_0] = :shipment
-                # """,
-                # shipment=self.sdhnum,
-            # ).first()
-
-    # def _fill_info_from_shipment(self):
-        # """
-        # When we learn the SOHNUM, we can copy information from the sales order.
-        # """
-        # result = self._get_shipment_from_x3()
-        # self.header.stofcy = result.STOFCY_0
-        # self.header.sdhnum = result.SDHNUM_0
-        # self.header.salfcy = result.SALFCY_0
-        # self.header.bpcord = result.BPCORD_0
-        # self.header.cur = result.CUR_0
-        # self.header.sohnum = result.SOHNUM_0
-
 
     def output(self, import_file: typing.TextIO):
         """

edi_997_inbound.py

@@ -22,7 +22,7 @@ import yamamotoyama.x3_imports  # type: ignore
 THIS_DIRECTORY = pathlib.Path(__file__).parent
 X12_DIRECTORY = THIS_DIRECTORY / "incoming"
 
-SHANDEX_997_FILENAME_RE = re.compile( #TODO FIX REGEX
+SHANDEX_997_FILENAME_RE = re.compile(
     r"\A 997_YAMAMOTOYAMA_ .* [.]edi \Z", re.X | re.M | re.S
 )
 
@@ -46,7 +46,7 @@ def tokens_from_edi_file(
     with edi_filename.open(encoding="utf-8", newline="") as edi_file:
         for record in edi_file.read().split("~"):
             fields = record.split("*")
-            if fields[0] in { #TODO see if there are more fields used in vendor EDI
+            if fields[0] in {
                 "ISA",
                 "GS",
                 "ST",

master_contoller.py

@@ -11,10 +11,12 @@ import edi_997_inbound
 import edi_944
 import edi_947
 import edi_846
-import edi_867
+import edi_867_to_table
 import edi_997_outbound
 import update_shandex_dashboard
 import edi_943
+import unprocessed_files_report
+import import_867s
 
 THIS_DIRECTORY = pathlib.Path(__file__).parent
 X12_SHANDEX_OUTGOING = THIS_DIRECTORY / "outgoing"
@@ -28,22 +30,29 @@ def main():
     #pick up files from Shandex
     retrieve_x12_edi_files_shandex()
     
+    #report on anything not handled
+    unprocessed_files_report.main()
+    
     #process all EDIs that started with Shandex
     edi_997_inbound.main()
     edi_944.main()
     edi_947.main()
-    #edi_846.main()
+    edi_846.main()
-    edi_867.main()
+    edi_867_to_table.main()
+    import_867s.main()
     
     # process all EDIs that start with us
     edi_943.main()
     edi_997_outbound.main()
     
     #send them to Shandex
-    send_x12_edi_files_shandex()#TODO production changes
+    send_x12_edi_files_shandex()
     
-    #update dashboard
+    #update dashboard - no longer needed now that it has been moved to the staging database 2024-08-20
-    # update_shandex_dashboard.main()
+    #update_shandex_dashboard.main()
+    
+    #report on anything not handled
+    unprocessed_files_report.main()
     
 
 
@@ -69,7 +78,7 @@ def send_x12_edi_files_shandex():
             hostname=SHANDEX_SFTP_HOST, username=SHANDEX_SFTP_USERNAME, password=SHANDEX_SFTP_PASSWORD
         )
         with ssh_client.open_sftp() as sftp_connection:
-            sftp_connection.chdir("/Stash/Test/ToShandex") #TODO change to production folder
+            sftp_connection.chdir("/Stash/Prod/ToShandex")
             for filename in X12_SHANDEX_OUTGOING.glob("*.edi"):
                 sftp_connection.put(filename, str(filename.name))
                 shutil.move(filename, X12_SHANDEX_OUTGOING / "archive" / filename.name)
@@ -84,11 +93,11 @@ def retrieve_x12_edi_files_shandex():
             hostname=SHANDEX_SFTP_HOST, username=SHANDEX_SFTP_USERNAME, password=SHANDEX_SFTP_PASSWORD
         )
         with ssh_client.open_sftp() as sftp_connection:
-            sftp_connection.chdir("/Stash/Test/FromShandex")
+            sftp_connection.chdir("/Stash/Prod/FromShandex")
             for filename in sftp_connection.listdir():
                 if filename.endswith(".edi"):
                     sftp_connection.get(filename, X12_SHANDEX_INCOMING / filename)
-                    new_filename = f"/Stash/Test/FromShandex/Archive/{filename}"
+                    new_filename = f"/Stash/Archive/{filename}"
                     sftp_connection.rename(filename, new_filename)
 
 

readme.txt

@@ -1,8 +1,5 @@
 Process EDI documents between YU and Shandex
 
------------------------------------------------------------
-EDI types accepted:
-
 997	Functional acknowledgment	Tell Shandex we received an EDI
 944	Stock transfer receipt	Receive qty and lots from replenishment shipment
 947	Inventory advice	Tell Stash of Q and R status inventory
@@ -10,15 +7,19 @@ EDI types accepted:
 867	Resale report	Tell Stash qty, price, and lot sold
 943	Stock transfer	Tell Shandex qty and lots being shipped
 
------------------------------------------------------------
-General operation:
 master_controller takes EDI files from the Shandex FTP, processes them, and loads our EDI files.
 
-Sales come in via the 867 script and the shandex_dashboard script stores which files have been received.
+Individual edi_***.py files process specific types of transactions.
-The corresponding control numbers can be matched to X3 Sales Deliveries in the YLICPLATE_0 field.
 
------------------------------------------------------------
+Sales come in via 867 and the shandex_dashboard script stores which files have been received.
-Currently runs in the 30 minute X3 recurring task
+The corresponding control numbers can be matched to X3 Sales Deliveries in the YLICPLATE_0 and YCLIPPERSHIP_0 fields.
 
-To switch between PROD and DEV, database connections need to be passed 'test'
+-------------------
-and the schema names in SQL queries need to be correct for the test folder being used.
+Issues
+-------------------
+867s will fail to process if a customer has not been mapped for them in the edi_867.py file.
+These problems are logged in the staging database in staging.dbo.shandex_shipments. To reprocess,
+add the customer code to the table and check it's is_sent field. Make sure to add the right code to
+the dictionary in 867 processing as well.
+
+944s need to be rerun in the edi_944.py script after validation, so that lots appear in the import file.

update_shandex_dashboard.py

@@ -7,18 +7,12 @@ To reprocess a file, move it to the correct folder on the FTP
 import shutil
 import yamamotoyama
 import pathlib
+import pprint
 
 THIS_DIRECTORY = pathlib.Path(__file__).parent
 RECEIVED_867_DIRECTORY = THIS_DIRECTORY / "processed_867s"
 ARCHIVE = RECEIVED_867_DIRECTORY / "archive"
 
-    
-INSERT_867s = """\
-    execute [PROD].[insert_shandex_867_data]
-    :ctrlnum,
-    :credat;
-    """
-    
 
 def main():
     """
@@ -41,14 +35,31 @@ def remove_completed_867s():
 def import_received_867s(data):
     with yamamotoyama.get_connection() as data_base:
         with data_base.transaction():
-            data_base.bulk_query(INSERT_867s, data)
+            data_base.query(
-
+            """
+            EXECUTE [PROD].[insert_shandex_867_data] 
+               :ctrlnum,
+               :ylicplate,
+               :credat,
+               :customer_po,
+               :shandex_so
+            """,
+            ctrlnum=data['ctrlnum']+'-'+data['ylicplate'],
+            ylicplate=data['ylicplate'],
+            credat=data['credat'],
+            customer_po=data['customer_po'],
+            shandex_so=data['shandex_so'],
+            )
 
 def process_received_867s():
-    #pull out the control number on the GS line
+    #pull out the control number on the GS line and the BOL# on PTD
     for file in RECEIVED_867_DIRECTORY.iterdir():
         control_number = ''
         transaction_date = ''
+        picking_number = ''
+        customer_po = ''
+        shandex_so = ''
+        data_set = []
         if file.name.endswith('.edi'):
             with file.open(encoding="utf-8", newline="") as edi_file:
                 for record in edi_file.read().split("~"):
@@ -57,11 +68,25 @@ def process_received_867s():
                         control_number = fields[6]
                     if fields[0] == 'BPT':
                         transaction_date = fields[3]
-                    data = {
+                    # REF*PO*3L49287E
-                        "ctrlnum":control_number,
+                    if fields[0] == 'REF' and fields[1] == 'PO':
-                        "credat":transaction_date
+                        customer_po = fields[2]
+                    # REF*IL*O0215276
+                    if fields[0] == 'REF' and fields[1] == 'IL':
+                        shandex_so = fields[2]
+                    if fields[0] == "PTD" and len(fields) > 2:
+                        picking_number = fields[5]
+                        data = {
+                            "ctrlnum":control_number,
+                            "ylicplate":picking_number,
+                            "credat":transaction_date,
+                            "customer_po":customer_po,
+                            "shandex_so":shandex_so,
                         }
-                import_received_867s(data)
+                        if data not in data_set:
+                            data_set.append(data)
+                            import_received_867s(data)
+                    
 
 
 if __name__ == "__main__":