bleeson
/
shandex_edi_2024
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Reworked 867 process and minor updates to others.

master
bleeson 2024-08-20 09:43:11 -07:00
parent 5e9de92626
commit 0a569a5b51
8 changed files with 1166 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
edi_846.py
View File

@ -49,7 +49,7 @@ def main():
if SHANDEX_846_FILENAME_RE.match(edi_filename.name):
shandex_inventory=process_file(edi_filename)
# file moved to 997 processing folder to be sent later
#shutil.move(edi_filename, EDI_997_DIRECTORY / edi_filename.name)
shutil.move(edi_filename, EDI_997_DIRECTORY / edi_filename.name)
#get stock information about WON and store it
#pass date from EDI so we can subtract newer stock movements?
x3_won_inventory=get_x3_won_inventory()
@ -214,7 +214,8 @@ def stock_count_alert():
msg['Subject'] = 'New Stock Count from Shandex'
msg['Precedence'] = 'bulk'
msg['From'] = 'x3report@stashtea.com'
msg['To'] = 'bleeson@stashtea.com'#TODO correct receipientscares
msg['To'] = 'isenn@yamamotoyama.com,vgomez@yamamotoyama.com'
msg['Cc'] = 'bleeson@stashtea.com'
emailtext = f'Attached.'
msg.attach(MIMEText(emailtext, 'plain'))
for file in EDI_846_ATTACHMENTS.iterdir():

1077
edi_867_to_table.py Normal file
View File

File diff suppressed because it is too large Load Diff

24
edi_943.py
View File

@ -19,7 +19,7 @@ import yamamotoyama # type:ignore
SITE_MAPPING = {
'WNJ' : 'SOURCELOGISTICS',
'WCA' : 'SOURCELOGISTICS',
'WON' : 'SHANDEXTEST ' # TODO CHANGE TO SHANDEX, needs to be 15 characters
'WON' : 'SHANDEX ' #Testing value is SHANDEXTEST with spaces to 15 characters
}
SHIPPING_CODE_MAPPING = {
@ -35,14 +35,12 @@ SHIPPING_CODE_MAPPING = {
THIS_DIRECTORY = pathlib.Path(__file__).parent
X12_SHANDEX = THIS_DIRECTORY / "outgoing"
# X12_SOURCELOGISTICS = THIS_DIRECTORY / "edi-testing" #test directories
# X12_SHANDEX = THIS_DIRECTORY / "edi-testing"
def main():
"""
Do it!
"""
with yamamotoyama.get_connection('test') as database:
with yamamotoyama.get_connection() as database:
shipments = list(get_shipments(database))
for shipment in shipments:
write_943(database, shipment)
@ -127,7 +125,7 @@ def write_943(database: records.Connection, shipment: str):
with database.transaction() as _:
database.query(
"""
update [FY23TEST].[SDELIVERY]
update [PROD].[SDELIVERY]
set [XX4S_943RDY_0] = 1
where [SOHNUM_0] = :shipment
""",
@ -136,7 +134,7 @@ def write_943(database: records.Connection, shipment: str):
order = get_order_for_shipment(database, shipment)
database.query(
"""
update [FY23TEST].[SORDER]
update [PROD].[SORDER]
set [XX4S_UDF2_0] = :sent_message
where [SOHNUM_0] = :order
""",
@ -153,7 +151,7 @@ def get_shipment_destination(database: records.Connection, shipment: str) -> str
"""
select
[SDH].[BPCORD_0]
from [FY23TEST].[SDELIVERY] as [SDH]
from [PROD].[SDELIVERY] as [SDH]
where
[SDH].[SDHNUM_0] = :shipment
""",
@ -172,7 +170,7 @@ def get_order_for_shipment(database: records.Connection, shipment: str) -> str:
"""
select
[SDH].[SOHNUM_0]
from [FY23TEST].[SDELIVERY] as [SDH]
from [PROD].[SDELIVERY] as [SDH]
where
[SDH].[SDHNUM_0] = :shipment
""",
@ -191,8 +189,8 @@ def get_shipments(database: records.Connection) -> typing.Iterator[str]:
"""
select
[SDH].[SDHNUM_0]
from [FY23TEST].[SDELIVERY] as [SDH]
join [FY23TEST].[SORDER] as [SOH]
from [PROD].[SDELIVERY] as [SDH]
join [PROD].[SORDER] as [SOH]
on [SOH].[SOHNUM_0] = [SDH].[SOHNUM_0]
where
(
@ -203,7 +201,7 @@ def get_shipments(database: records.Connection) -> typing.Iterator[str]:
)
and [SDH].[SHIDAT_0] >= {d'2023-10-09'}
and (
[SOH].[XX4S_UDF2_0] = ''
[SOH].[XX4S_UDF2_0] not like '943%'
or [SDH].[XX4S_943RDY_0] = 2
)
and (
@ -267,7 +265,7 @@ def get_shipment(
,[GROWEI_0]
,[CFMFLG_0]
,[SHIDAT_0]
from [FY23TEST].[zyumiddleware_shipment] as [SDH]
from [PROD].[zyumiddleware_shipment_shandex] as [SDH]
where
[SDH].[SDHNUM_0] = :shipment
""",
@ -549,7 +547,7 @@ class ShipmentDetail(X12):
"",
"",
self.gtin_or_upc_marker,
self.gtin_or_upc_code,#W04-15
self.gtin_or_upc_code.replace(' ',''),#W04-15
]
),
self.line(

75
edi_947.py
View File

@ -34,10 +34,6 @@ X12_DIRECTORY = THIS_DIRECTORY / "incoming"
IMPORTS_DIRECTORY = THIS_DIRECTORY / "x3_imports"
EDI_997_DIRECTORY = THIS_DIRECTORY / "997_processing"
SHANDEX_947_FILENAME_RE = re.compile(
r"\A 947_YAMAMOTOYAMA_ \S+ [.]edi \Z", re.X | re.M | re.S
)
#TODO remove this and have a single production name? why is this one different
SHANDEX_947_FILENAME_RE = re.compile(
r"\A 947_QTY_STASH-YAMAMOTOYAMA_ \S+ [.]edi \Z", re.X | re.M | re.S
)
@ -80,7 +76,7 @@ def main():
if SHANDEX_947_FILENAME_RE.match(edi_filename.name):
process_file(edi_filename)
# file moved to 997 processing folder to be sent later
#shutil.move(edi_filename, EDI_997_DIRECTORY / edi_filename.name)
shutil.move(edi_filename, EDI_997_DIRECTORY / edi_filename.name)
combine_zscs()
@ -154,7 +150,8 @@ def stock_movement_alert(itmref, qty, lot, status):
msg['Subject'] = 'New Stock Change from Shandex'
msg['Precedence'] = 'bulk'
msg['From'] = 'x3report@stashtea.com'
msg['To'] = 'bleeson@stashtea.com'#TODO correct receipientscares
msg['To'] = 'isenn@yamamotoyama.com, vgomez@yamamotoyama.com'
msg['CC'] = 'bleeson@stashtea.com'
emailtext = f'Item: {itmref}\nQty: {qty}\nLot: {lot}\nStatus: {DAMAGE_CODE_MAPPING[status]}\nReason: {DAMAGE_CODE_DESCRIPTIONS_MAPPING[status]}'
msg.attach(MIMEText(emailtext, 'plain'))
with smtplib.SMTP_SSL("smtp.gmail.com", 465) as smtp:
@ -222,37 +219,6 @@ class StockChangeSubDetail:
loc: str = ""
sta: str = "A"
# def stojous(self, shipment, item) -> typing.List[str]:
# """
# Convert grouped lot quantities into individual STOJOU records to fit on stockchange
# """
# with yamamotoyama.get_connection('test') as database: #todo remove 'test'
# details = (
# database.query(
# """
# select
# 'S',
# 'A',
# [STJ].[PCU_0],
# cast(cast(-1*[STJ].[QTYSTU_0] as int) as nvarchar),
# [STJ].[LOT_0],
# '',
# ''
# from [FY23TEST].[STOJOU] [STJ] --TODO change to PROD
# where
# [STJ].[VCRNUM_0] = :sdhnum
# and [STJ].[ITMREF_0] = :itmref
# and [STJ].[LOT_0] = :lot
# and [STJ].[TRSTYP_0] = 4
# """,
# sdhnum=shipment,
# itmref=item,
# lot=self.lot,
# )
# .all()
# )
# return details
def convert_to_strings(self) -> typing.List[str]:
"""
@ -288,7 +254,7 @@ class StockChangeDetail:
itmref: str = ""
pcu: str = ""
pcustucoe: int = 1 #does this need a lookup?
sta: str = "A" #todo this needs to flip based on the transaction A > R, A > Q, what about Q > A?
sta: str = "A"
loctyp: str = ""
loc: str = ""
lot: str = ""
@ -564,39 +530,6 @@ class StockChange:
if value:
self._fill_info_from_shipment()
# def _get_shipment_from_x3(self) -> records.Record:
# """
# Fetch shipment from X3 database.
# """
# with yamamotoyama.get_connection('test') as db_connection:#todo remove 'test'
# return db_connection.query(
# """
# select
# [SDH].[STOFCY_0],
# [SDH].[SDHNUM_0],
# [SDH].[SALFCY_0],
# [SDH].[BPCORD_0],
# [SDH].[CUR_0],
# [SDH].[SOHNUM_0]
# from [FY23TEST].[SDELIVERY] [SDH]--TODO change back to [PROD]
# where
# [SDH].[SDHNUM_0] = :shipment
# """,
# shipment=self.sdhnum,
# ).first()
# def _fill_info_from_shipment(self):
# """
# When we learn the SOHNUM, we can copy information from the sales order.
# """
# result = self._get_shipment_from_x3()
# self.header.stofcy = result.STOFCY_0
# self.header.sdhnum = result.SDHNUM_0
# self.header.salfcy = result.SALFCY_0
# self.header.bpcord = result.BPCORD_0
# self.header.cur = result.CUR_0
# self.header.sohnum = result.SOHNUM_0
def output(self, import_file: typing.TextIO):
"""

4
edi_997_inbound.py
View File

@ -22,7 +22,7 @@ import yamamotoyama.x3_imports # type: ignore
THIS_DIRECTORY = pathlib.Path(__file__).parent
X12_DIRECTORY = THIS_DIRECTORY / "incoming"
SHANDEX_997_FILENAME_RE = re.compile( #TODO FIX REGEX
SHANDEX_997_FILENAME_RE = re.compile(
r"\A 997_YAMAMOTOYAMA_ .* [.]edi \Z", re.X | re.M | re.S
)
@ -46,7 +46,7 @@ def tokens_from_edi_file(
with edi_filename.open(encoding="utf-8", newline="") as edi_file:
for record in edi_file.read().split("~"):
fields = record.split("*")
if fields[0] in { #TODO see if there are more fields used in vendor EDI
if fields[0] in {
"ISA",
"GS",
"ST",

27
master_contoller.py
View File

@ -11,10 +11,12 @@ import edi_997_inbound
import edi_944
import edi_947
import edi_846
import edi_867
import edi_867_to_table
import edi_997_outbound
import update_shandex_dashboard
import edi_943
import unprocessed_files_report
import import_867s
THIS_DIRECTORY = pathlib.Path(__file__).parent
X12_SHANDEX_OUTGOING = THIS_DIRECTORY / "outgoing"
@ -28,22 +30,29 @@ def main():
#pick up files from Shandex
retrieve_x12_edi_files_shandex()
#report on anything not handled
unprocessed_files_report.main()
#process all EDIs that started with Shandex
edi_997_inbound.main()
edi_944.main()
edi_947.main()
#edi_846.main()
edi_867.main()
edi_846.main()
edi_867_to_table.main()
import_867s.main()
# process all EDIs that start with us
edi_943.main()
edi_997_outbound.main()
#send them to Shandex
send_x12_edi_files_shandex()#TODO production changes
send_x12_edi_files_shandex()
#update dashboard
# update_shandex_dashboard.main()
#update dashboard - no longer needed now that it has been moved to the staging database 2024-08-20
#update_shandex_dashboard.main()
#report on anything not handled
unprocessed_files_report.main()
@ -69,7 +78,7 @@ def send_x12_edi_files_shandex():
hostname=SHANDEX_SFTP_HOST, username=SHANDEX_SFTP_USERNAME, password=SHANDEX_SFTP_PASSWORD
)
with ssh_client.open_sftp() as sftp_connection:
sftp_connection.chdir("/Stash/Test/ToShandex") #TODO change to production folder
sftp_connection.chdir("/Stash/Prod/ToShandex")
for filename in X12_SHANDEX_OUTGOING.glob("*.edi"):
sftp_connection.put(filename, str(filename.name))
shutil.move(filename, X12_SHANDEX_OUTGOING / "archive" / filename.name)
@ -84,11 +93,11 @@ def retrieve_x12_edi_files_shandex():
hostname=SHANDEX_SFTP_HOST, username=SHANDEX_SFTP_USERNAME, password=SHANDEX_SFTP_PASSWORD
)
with ssh_client.open_sftp() as sftp_connection:
sftp_connection.chdir("/Stash/Test/FromShandex")
sftp_connection.chdir("/Stash/Prod/FromShandex")
for filename in sftp_connection.listdir():
if filename.endswith(".edi"):
sftp_connection.get(filename, X12_SHANDEX_INCOMING / filename)
new_filename = f"/Stash/Test/FromShandex/Archive/{filename}"
new_filename = f"/Stash/Archive/{filename}"
sftp_connection.rename(filename, new_filename)

23
readme.txt
View File

@ -1,8 +1,5 @@
Process EDI documents between YU and Shandex
-----------------------------------------------------------
EDI types accepted:
997 Functional acknowledgment Tell Shandex we received an EDI
944 Stock transfer receipt Receive qty and lots from replenishment shipment
947 Inventory advice Tell Stash of Q and R status inventory
@ -10,15 +7,19 @@ EDI types accepted:
867 Resale report Tell Stash qty, price, and lot sold
943 Stock transfer Tell Shandex qty and lots being shipped
-----------------------------------------------------------
General operation:
master_controller takes EDI files from the Shandex FTP, processes them, and loads our EDI files.
Sales come in via the 867 script and the shandex_dashboard script stores which files have been received.
The corresponding control numbers can be matched to X3 Sales Deliveries in the YLICPLATE_0 field.
Individual edi_***.py files process specific types of transactions.
-----------------------------------------------------------
Currently runs in the 30 minute X3 recurring task
Sales come in via 867 and the shandex_dashboard script stores which files have been received.
The corresponding control numbers can be matched to X3 Sales Deliveries in the YLICPLATE_0 and YCLIPPERSHIP_0 fields.
To switch between PROD and DEV, database connections need to be passed 'test'
and the schema names in SQL queries need to be correct for the test folder being used.
-------------------
Issues
-------------------
867s will fail to process if a customer has not been mapped for them in the edi_867.py file.
These problems are logged in the staging database in staging.dbo.shandex_shipments. To reprocess,
add the customer code to the table and check it's is_sent field. Make sure to add the right code to
the dictionary in 867 processing as well.
944s need to be rerun in the edi_944.py script after validation, so that lots appear in the import file.

47
update_shandex_dashboard.py
View File

@ -7,19 +7,13 @@ To reprocess a file, move it to the correct folder on the FTP
import shutil
import yamamotoyama
import pathlib
import pprint
THIS_DIRECTORY = pathlib.Path(__file__).parent
RECEIVED_867_DIRECTORY = THIS_DIRECTORY / "processed_867s"
ARCHIVE = RECEIVED_867_DIRECTORY / "archive"
INSERT_867s = """\
execute [PROD].[insert_shandex_867_data]
:ctrlnum,
:credat;
"""
def main():
"""
Do it!
@ -41,14 +35,31 @@ def remove_completed_867s():
def import_received_867s(data):
with yamamotoyama.get_connection() as data_base:
with data_base.transaction():
data_base.bulk_query(INSERT_867s, data)
data_base.query(
"""
EXECUTE [PROD].[insert_shandex_867_data]
:ctrlnum,
:ylicplate,
:credat,
:customer_po,
:shandex_so
""",
ctrlnum=data['ctrlnum']+'-'+data['ylicplate'],
ylicplate=data['ylicplate'],
credat=data['credat'],
customer_po=data['customer_po'],
shandex_so=data['shandex_so'],
)
def process_received_867s():
#pull out the control number on the GS line
#pull out the control number on the GS line and the BOL# on PTD
for file in RECEIVED_867_DIRECTORY.iterdir():
control_number = ''
transaction_date = ''
picking_number = ''
customer_po = ''
shandex_so = ''
data_set = []
if file.name.endswith('.edi'):
with file.open(encoding="utf-8", newline="") as edi_file:
for record in edi_file.read().split("~"):
@ -57,12 +68,26 @@ def process_received_867s():
control_number = fields[6]
if fields[0] == 'BPT':
transaction_date = fields[3]
# REF*PO*3L49287E
if fields[0] == 'REF' and fields[1] == 'PO':
customer_po = fields[2]
# REF*IL*O0215276
if fields[0] == 'REF' and fields[1] == 'IL':
shandex_so = fields[2]
if fields[0] == "PTD" and len(fields) > 2:
picking_number = fields[5]
data = {
"ctrlnum":control_number,
"credat":transaction_date
"ylicplate":picking_number,
"credat":transaction_date,
"customer_po":customer_po,
"shandex_so":shandex_so,
}
if data not in data_set:
data_set.append(data)
import_received_867s(data)
if __name__ == "__main__":
main()