112 lines
3.6 KiB
Python
112 lines
3.6 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
"""
|
||
|
|
Bring in Shandex SO inforamtion from the FTP
|
||
|
|
into the staging database
|
||
|
|
"""
|
||
|
|
import contextlib
|
||
|
|
import shutil
|
||
|
|
import pathlib
|
||
|
|
import paramiko # type: ignore
|
||
|
|
import csv
|
||
|
|
import pprint
|
||
|
|
import yamamotoyama
|
||
|
|
|
||
|
|
THIS_DIRECTORY = pathlib.Path(__file__).parent
|
||
|
|
INCOMING_DIRECTORY = THIS_DIRECTORY / "incoming"
|
||
|
|
|
||
|
|
SSH_DIRECTORY = THIS_DIRECTORY / "ssh"
|
||
|
|
SSH_KNOWN_HOSTS_FILE = str(SSH_DIRECTORY / "known_hosts")
|
||
|
|
SSH_KEY_FILENAME = str(SSH_DIRECTORY / "id_ed25519")
|
||
|
|
|
||
|
|
SHANDEX_SFTP_HOST = "ftp.shandex.com"
|
||
|
|
SHANDEX_SFTP_USERNAME = "Stash"
|
||
|
|
SHANDEX_SFTP_PASSWORD = "ST@Pass2024$$"
|
||
|
|
|
||
|
|
INSERT_ORDER = """
|
||
|
|
EXECUTE [staging].[dbo].[shandex_insert_order_detail]
|
||
|
|
:customer_order_number
|
||
|
|
,:customer_po
|
||
|
|
,:Order_Type
|
||
|
|
,:order_status
|
||
|
|
,:import_date
|
||
|
|
,:RSD
|
||
|
|
,:ship_date
|
||
|
|
,:customer_number
|
||
|
|
,:sold_to_name
|
||
|
|
,:line_number
|
||
|
|
,:supplier_part_number
|
||
|
|
,:sku
|
||
|
|
,:description
|
||
|
|
,:qty_ordered
|
||
|
|
,:qty_picked
|
||
|
|
,:qty_shorted
|
||
|
|
,:uom
|
||
|
|
,:unit_price
|
||
|
|
"""
|
||
|
|
|
||
|
|
def main():
|
||
|
|
"""
|
||
|
|
Do it!
|
||
|
|
"""
|
||
|
|
#pick up files from Shandex
|
||
|
|
retrieve_x12_edi_files_shandex()
|
||
|
|
for file in INCOMING_DIRECTORY.iterdir():
|
||
|
|
if file.name.endswith(".csv"):
|
||
|
|
process_file(file)
|
||
|
|
shutil.move(file, INCOMING_DIRECTORY / "archive" / file.name)
|
||
|
|
|
||
|
|
def retrieve_x12_edi_files_shandex():
|
||
|
|
with paramiko.SSHClient() as ssh_client:
|
||
|
|
ssh_client.load_system_host_keys()
|
||
|
|
ssh_client.load_host_keys(SSH_KNOWN_HOSTS_FILE)
|
||
|
|
ssh_client.set_missing_host_key_policy(paramiko.client.RejectPolicy)
|
||
|
|
ssh_client.connect(
|
||
|
|
hostname=SHANDEX_SFTP_HOST, username=SHANDEX_SFTP_USERNAME, password=SHANDEX_SFTP_PASSWORD
|
||
|
|
)
|
||
|
|
with ssh_client.open_sftp() as sftp_connection:
|
||
|
|
sftp_connection.chdir("/Stash/Test/FromShandex") #TODO update this to Production folder
|
||
|
|
for filename in sftp_connection.listdir():
|
||
|
|
if filename.startswith("Stash_Tea_Order_Data"):
|
||
|
|
sftp_connection.get(filename, INCOMING_DIRECTORY / filename)
|
||
|
|
new_filename = f"/Stash/Test/FromShandex/Archive/{filename}"
|
||
|
|
sftp_connection.rename(filename, new_filename)
|
||
|
|
|
||
|
|
def process_file(file):
|
||
|
|
#open file
|
||
|
|
all_data = []
|
||
|
|
with file.open(encoding="utf-8", newline="") as csv_file:
|
||
|
|
reader = csv.reader(csv_file)
|
||
|
|
headers = next(reader, None)#skip the header
|
||
|
|
for row in reader:
|
||
|
|
if row:
|
||
|
|
key_dict = {
|
||
|
|
"customer_order_number":row[0],
|
||
|
|
"customer_po":row[1],
|
||
|
|
"Order_Type":row[2],
|
||
|
|
"order_status":row[3],
|
||
|
|
"import_date":row[4],
|
||
|
|
"RSD":row[5],
|
||
|
|
"ship_date":row[6],
|
||
|
|
"customer_number":row[7],
|
||
|
|
"sold_to_name":row[8],
|
||
|
|
"line_number":row[9],
|
||
|
|
"supplier_part_number":row[10],
|
||
|
|
"sku":row[11],
|
||
|
|
"description":row[12],
|
||
|
|
"qty_ordered":row[13],
|
||
|
|
"qty_picked":row[14],
|
||
|
|
"qty_shorted":row[15],
|
||
|
|
"uom":row[16],
|
||
|
|
"unit_price":row[17],
|
||
|
|
}
|
||
|
|
all_data.append(key_dict)
|
||
|
|
if(all_data):
|
||
|
|
with yamamotoyama.get_connection() as data_base:
|
||
|
|
with data_base.transaction():
|
||
|
|
data_base.bulk_query(INSERT_ORDER, all_data)
|
||
|
|
#load vars and insert
|
||
|
|
#archive
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
main()
|