bleeson
/
source_logistics_ecommerce_edi_2024
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

file movement fixed to match recurring tasks in X3.

master
bleeson 2024-03-07 11:02:22 -08:00
parent d1c161deb9
commit d119d9936f
3 changed files with 123 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
readme.txt
View File

@ -19,4 +19,7 @@ step 1 read in file
---------------------------------------------------------------------------------------- ----------------------------------------------------------------------------------------
TODO: schedule X3 batch server to process orders and then deliveries, but when do we receive files, how often should this be happening?
see ZECMSL recurring task

56
source_ecommerce_make_order.py
View File

@ -1,4 +1,12 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
"""
Pull shipment files from the Stash AWS FTP.
After copying file, move file to archive folder on FTP.
Create ZSHPORD import files, which X3 will consume on a schedule via ZECMSL recurring task.
Copy file to the archive and then pass it to the shipment maker directory.
TODO: Source needs to send us real data for final adjustments (site, discount, multilot, etc.)
"""
import csv import csv
import pprint import pprint
import dataclasses import dataclasses
@ -9,28 +17,35 @@ import pathlib
import re import re
import shutil import shutil
import typing import typing
import paramiko
import records # type: ignore import records # type: ignore
import yamamotoyama # type: ignore import yamamotoyama # type: ignore
import yamamotoyama.x3_imports # type: ignore import yamamotoyama.x3_imports # type: ignore
THIS_DIRECTORY = pathlib.Path(__file__).parent
SFTP_HOST = "s-8ade4d252cc44c50b.server.transfer.us-west-1.amazonaws.com" SFTP_HOST = "s-8ade4d252cc44c50b.server.transfer.us-west-1.amazonaws.com"
SFTP_USERNAME = "yumiddleware2023" SFTP_USERNAME = "yumiddleware2023"
SSH_DIRECTORY = edi_940.THIS_DIRECTORY / "ssh" #TODO fixme SSH_DIRECTORY = THIS_DIRECTORY / "ssh" #TODO fixme
SSH_KNOWN_HOSTS_FILE = str(SSH_DIRECTORY / "known_hosts") SSH_KNOWN_HOSTS_FILE = str(SSH_DIRECTORY / "known_hosts")
SSH_KEY_FILENAME = str(SSH_DIRECTORY / "id_ed25519") SSH_KEY_FILENAME = str(SSH_DIRECTORY / "id_ed25519")
THIS_DIRECTORY = pathlib.Path(__file__).parent
INCOMING_DIRECTORY = THIS_DIRECTORY / "incoming_orders" INCOMING_DIRECTORY = THIS_DIRECTORY / "incoming_orders"
SHIPMENTS_DIRECTORY = THIS_DIRECTORY / "incoming_shipments"
SOH_IMPORT_DIRECTORY = THIS_DIRECTORY / "to_import_SOH" SOH_IMPORT_DIRECTORY = THIS_DIRECTORY / "to_import_SOH"
def main(): def main():
retrieve_x12_edi_files() #retrieve_x12_edi_files()#TODO uncomment
for file in INCOMING_DIRECTORY.iterdir(): for file in INCOMING_DIRECTORY.iterdir():
process_files() if file.name[-4:] != '.csv':
shutil.move(file, INCOMING_DIRECTORY / "archive" / file.name) continue
combine_zshpords() else:
process_files(file)
shutil.move(file, SHIPMENTS_DIRECTORY / file.name)
#shutil.move(file, INCOMING_DIRECTORY / "archive" / file.name)#archives go in the shipping folder
combine_zshpords()
#TODO determine X3 processing schedule #TODO determine X3 processing schedule
def sftp_server() -> paramiko.SFTPClient: def sftp_server() -> paramiko.SFTPClient:
@ -48,11 +63,18 @@ def retrieve_x12_edi_files():
""" """
Connect to S3 bucket & pull down files. Connect to S3 bucket & pull down files.
""" """
with sftp_server() as sftp_connection: with paramiko.SSHClient() as ssh_client:
sftp_connection.chdir("/yu-edi-transfer/source-logi/dev/ecomm-inbound")#TODO set to prod ssh_client.load_system_host_keys()
for filename in sftp_connection.listdir(): ssh_client.load_host_keys(SSH_KNOWN_HOSTS_FILE)
if edi_945.SOURCE_945_FILENAME_RE.match(filename): ssh_client.set_missing_host_key_policy(paramiko.client.RejectPolicy)
sftp_connection.get(filename, edi_945.X12_DIRECTORY / filename) ssh_client.connect(
hostname=SFTP_HOST, username=SFTP_USERNAME, key_filename=SSH_KEY_FILENAME
)
with ssh_client.open_sftp() as sftp_connection:
sftp_connection.chdir("/yu-edi-transfer/source-logi/dev/ecomm-inbound")#TODO set to prod
for filename in sftp_connection.listdir():
#if edi_945.SOURCE_945_FILENAME_RE.match(filename):#TODO fixme
sftp_connection.get(filename, INCOMING_DIRECTORY / filename)
new_filename = f"/yu-edi-transfer/source-logi/dev/ecomm-processed/{filename}"#TODO set to prod new_filename = f"/yu-edi-transfer/source-logi/dev/ecomm-processed/{filename}"#TODO set to prod
sftp_connection.rename(filename, new_filename) sftp_connection.rename(filename, new_filename)
@ -60,12 +82,12 @@ def combine_zshpords():
""" """
Collect all ZSHPORD imports into a single file for easy import. Collect all ZSHPORD imports into a single file for easy import.
""" """
archive_directory = INCOMING_DIRECTORY / "archive" archive_directory = SOH_IMPORT_DIRECTORY / "archive"
archive_directory.mkdir(exist_ok=True) archive_directory.mkdir(exist_ok=True)
with (INCOMING_DIRECTORY / "ZSHPORD.dat").open( with (SOH_IMPORT_DIRECTORY / "ZSHPORD.dat").open(
"a", encoding="utf-8", newline="\n" "a", encoding="utf-8", newline="\n"
) as combined_import_file: ) as combined_import_file:
for individual_import_filename in INCOMING_DIRECTORY.glob( for individual_import_filename in SOH_IMPORT_DIRECTORY.glob(
"ZSHPORD_*.dat" "ZSHPORD_*.dat"
): ):
with individual_import_filename.open( with individual_import_filename.open(
@ -106,8 +128,8 @@ def process_files(file):
sales_order.header.stofcy = ship_site sales_order.header.stofcy = ship_site
sales_order.header.bpdnam = customer_name sales_order.header.bpdnam = customer_name
sales_order.header.invdtaamt_5 = ship_charge sales_order.header.invdtaamt_5 = ship_charge
sales_order.header.invdtaamt_7 = '0.33' #discount sales_order.header.invdtaamt_7 = '0.33' #TODO discount
sales_order.header.invdtaamt_8 = '0.51'#taxes sales_order.header.invdtaamt_8 = '0.51'#TODO taxes
#gather line data #gather line data
line_product = row[0] line_product = row[0]
@ -123,7 +145,7 @@ def process_files(file):
) )
time_stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") time_stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
with yamamotoyama.x3_imports.open_import_file( with yamamotoyama.x3_imports.open_import_file(
SOH_IMPORT_DIRECTORY / f"ZSHPORD_{time_stamp}.dat" SOH_IMPORT_DIRECTORY / f"ZSHPORD_{file.name}_{time_stamp}.dat"
) as import_file: ) as import_file:
sales_order.output(import_file) sales_order.output(import_file)

135
source_ecommerce_make_shipment.py
View File

@ -20,9 +20,37 @@ INCOMING_DIRECTORY = THIS_DIRECTORY / "incoming_shipments"
SDH_IMPORT_DIRECTORY = THIS_DIRECTORY / "to_import_SDH" SDH_IMPORT_DIRECTORY = THIS_DIRECTORY / "to_import_SDH"
def main(): def main():
process_files() for file in INCOMING_DIRECTORY.iterdir():
#TODO shutil the files around, archive on the ftp if file.name[-4:] != '.csv':
#TODO determine X3 processing schedule continue
else:
process_files(file)
shutil.move(file, INCOMING_DIRECTORY / "archive" / file.name)
combine_zship945s()
#TODO determine X3 processing schedule
#TODO determine X3 processing schedule
def combine_zship945s():
"""
Collect all ZSHPORD imports into a single file for easy import.
"""
archive_directory = SDH_IMPORT_DIRECTORY / "archive"
archive_directory.mkdir(exist_ok=True)
with (SDH_IMPORT_DIRECTORY / "ZSHIP945S.dat").open(
"a", encoding="utf-8", newline="\n"
) as combined_import_file:
for individual_import_filename in SDH_IMPORT_DIRECTORY.glob(
"ZSHIP945S_*.dat"
):
with individual_import_filename.open(
"r", encoding="utf-8", newline="\n"
) as individual_import_file:
for line in individual_import_file:
combined_import_file.write(line)
shutil.move(
individual_import_filename,
archive_directory / individual_import_filename.name,
)
def find_so_from_po(cust_po): def find_so_from_po(cust_po):
with yamamotoyama.get_connection('test') as db_connection:#TODO remove 'test' with yamamotoyama.get_connection('test') as db_connection:#TODO remove 'test'
@ -39,57 +67,56 @@ def find_so_from_po(cust_po):
order=cust_po, order=cust_po,
).first()["SOHNUM_0"] ).first()["SOHNUM_0"]
def process_files(): def process_files(file):
for file in INCOMING_DIRECTORY.iterdir(): with open(file) as source_file:
with open(file) as source_file: csv_reader = csv.reader(source_file)
csv_reader = csv.reader(source_file) warehouse_shipment = WarehouseShipment()
warehouse_shipment = WarehouseShipment() for num, row in enumerate(csv_reader):
for num, row in enumerate(csv_reader): if num == 0:
if num == 0: continue #skip header lines
continue #skip header lines if num == 1: #gather header information
if num == 1: #gather header information sohnum = find_so_from_po(row[5])
sohnum = find_so_from_po(row[5]) order_date = row[8]
order_date = row[8] customer_name = row[9]
customer_name = row[9] # shipadd1 = row[9] # address information is not stored in X3
# shipadd1 = row[9] # address information is not stored in X3 # shipadd2 = row[10]
# shipadd2 = row[10] # shipcity = row[11]
# shipcity = row[11] # shipstate = row[12]
# shipstate = row[12] # shipzip = row[13]
# shipzip = row[13] tracking = row[14]
tracking = row[14] weight = row[16]
weight = row[16] ship_charge = row[20]
ship_charge = row[20] taxes = "?" #TODO fixme
taxes = "?" #TODO fixme ship_site = "?" #TODO fixme
ship_site = "?" #TODO fixme discount = "?" #TODO fixme
discount = "?" #TODO fixme warehouse_shipment.sohnum = sohnum
warehouse_shipment.sohnum = sohnum #warehouse_shipment.header.sohnum = sohnum
#warehouse_shipment.header.sohnum = sohnum warehouse_shipment.header.shidat = datetime.datetime.strptime(order_date,'%m/%d/%Y')
warehouse_shipment.header.shidat = datetime.datetime.strptime(order_date,'%m/%d/%Y') warehouse_shipment.header.ylicplate = tracking
warehouse_shipment.header.ylicplate = tracking warehouse_shipment.header.growei = weight
warehouse_shipment.header.growei = weight #gather line data
#gather line data #TODO how are multiple lots processed?
#TODO how are multiple lots processed? line_product = row[0]
line_product = row[0] line_qty = row[2]
line_qty = row[2] line_lot = row[3]
line_lot = row[3] line_price = row[19]
line_price = row[19] subdetail = WarehouseShipmentSubDetail(
subdetail = WarehouseShipmentSubDetail( qtypcu=-1 * int(line_qty),
qtypcu=-1 * int(line_qty), lot=line_lot,
lot=line_lot, )
) warehouse_shipment.append(
warehouse_shipment.append( WarehouseShipmentDetail(
WarehouseShipmentDetail( sohnum=sohnum,
sohnum=sohnum, itmref=line_product,
itmref=line_product, qty=int(line_qty),
qty=int(line_qty), ),
), subdetail,
subdetail, )
) time_stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
time_stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") with yamamotoyama.x3_imports.open_import_file(
with yamamotoyama.x3_imports.open_import_file( SDH_IMPORT_DIRECTORY / f"ZSHIP945S_{warehouse_shipment.sohnum}_{time_stamp}.dat"
SDH_IMPORT_DIRECTORY / f"ZSHIP945S_{warehouse_shipment.sohnum}_{time_stamp}.dat" ) as import_file:
) as import_file: warehouse_shipment.output(import_file)
warehouse_shipment.output(import_file)
@dataclasses.dataclass @dataclasses.dataclass
class WarehouseShipmentSubDetail: class WarehouseShipmentSubDetail: