From 30e028609c861a90ae15915215b5f56fb0031afa Mon Sep 17 00:00:00 2001 From: bleeson Date: Mon, 5 May 2025 15:21:23 -0700 Subject: [PATCH] Google auth changes and to_table work --- credentials.json | 1 + edi_846.py | 54 ++- edi_850.py | 52 ++- edi_867_to_table.py | 61 +++- edi_944.py | 63 +++- edi_944_to_table.py | 663 +++++++++++++++++++++++++++++++++++ edi_947.py | 56 ++- import_944s.py | 221 ++++++++++++ master_contoller.py | 12 +- reimport_from_archive.py | 55 +++ simple_email_notification.py | 70 ++++ unprocessed_files_report.py | 41 +++ 12 files changed, 1319 insertions(+), 30 deletions(-) create mode 100644 credentials.json create mode 100644 edi_944_to_table.py create mode 100644 import_944s.py create mode 100644 reimport_from_archive.py create mode 100644 simple_email_notification.py create mode 100644 unprocessed_files_report.py diff --git a/credentials.json b/credentials.json new file mode 100644 index 0000000..a0c9505 --- /dev/null +++ b/credentials.json @@ -0,0 +1 @@ +{"installed":{"client_id":"249203512502-8ut4vkh77ns4rl40ia485t460niii2b8.apps.googleusercontent.com","project_id":"python-access-2025","auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://oauth2.googleapis.com/token","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs","client_secret":"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","redirect_uris":["http://localhost"]}} \ No newline at end of file diff --git a/edi_846.py b/edi_846.py index 60b75ff..a1665d7 100644 --- a/edi_846.py +++ b/edi_846.py @@ -17,11 +17,22 @@ from email.mime.multipart import MIMEMultipart from email.mime.application import MIMEApplication from email.mime.text import MIMEText +import os +import base64 +import google.auth +import pickle +# Gmail API utils +from googleapiclient.discovery import build +from google_auth_oauthlib.flow import InstalledAppFlow +from google.auth.transport.requests import Request + import records # type: ignore import yamamotoyama # type: ignore import yamamotoyama.x3_imports # type: ignore +SCOPES = ['https://mail.google.com/'] + THIS_DIRECTORY = pathlib.Path(__file__).parent X12_DIRECTORY = THIS_DIRECTORY / "incoming" IMPORTS_DIRECTORY = THIS_DIRECTORY / "x3_imports" @@ -59,6 +70,38 @@ def main(): stock_count_alert() +def gmail_authenticate(): + creds = None + # the file token.pickle stores the user's access and refresh tokens, and is + # created automatically when the authorization flow completes for the first time + if os.path.exists("token.pickle"): + with open("token.pickle", "rb") as token: + creds = pickle.load(token) + # if there are no (valid) credentials availablle, let the user log in. + if not creds or not creds.valid: + if creds and creds.expired and creds.refresh_token: + creds.refresh(Request()) + else: + flow = InstalledAppFlow.from_client_secrets_file('credentials.json', SCOPES) + creds = flow.run_local_server(port=0) + # save the credentials for the next run + with open("token.pickle", "wb") as token: + pickle.dump(creds, token) + return build('gmail', 'v1', credentials=creds) + + +def gmail_send_message(service, payload): + create_message = {"raw": payload} + # pylint: disable=E1101 + send_message = ( + service.users() + .messages() + .send(userId="me", body=create_message) + .execute() + ) + return send_message + + def compare_inventory(shandex_inventory, x3_inventory): today = datetime.datetime.today() today = today.strftime('%Y-%m-%d') @@ -215,7 +258,7 @@ def stock_count_alert(): msg['Subject'] = 'New Stock Count from Shandex' msg['Precedence'] = 'bulk' msg['From'] = 'x3report@stashtea.com' - msg['To'] = 'isenn@yamamotoyama.com,vgomez@yamamotoyama.com' + msg['To'] = 'jpena@yamamotoyama.com,icortes@yamamotoyama.com,mdelacruz@yamamotoyama.com' msg['Cc'] = 'bleeson@stashtea.com' emailtext = f'Attached.' msg.attach(MIMEText(emailtext, 'plain')) @@ -225,9 +268,12 @@ def stock_count_alert(): part['Content-Disposition'] = f'attachment; filename="{file.name}"' msg.attach(part) shutil.move(file, EDI_846_ATTACHMENTS_ARCHIVE / file.name) - with smtplib.SMTP_SSL("smtp.gmail.com", 465) as smtp: - smtp.login(user='x3reportmk2@yamamotoyama.com', password=r'n typing.Iterator[typing.List[str]]: + """ + Read tokens from EDI file + """ + with edi_filename.open(encoding="utf-8", newline="") as edi_file: + for record in edi_file.read().split("~"): + fields = record.split("*") + if fields[0] in { + "ISA", + "ST", + "N2", + "N3", + "N4", + "LX", + }: + continue + yield fields + + +def find_shipment_line(sdhnum, itmref): + with yamamotoyama.get_connection() as database: + result = database.query( + """ + select + SDDLIN_0 + from PROD.SDELIVERYD + where + SDHNUM_0 = :sdhnum + and ITMREF_0 = :itmref + """, + sdhnum=sdhnum, + itmref=itmref + ).first()['SDDLIN_0'] + return result + + +def check_shipment_status(delivery): + with yamamotoyama.get_connection() as database: + result = database.query( + """ + select + SDH.SDHNUM_0, + CFMFLG_0 + from PROD.SDELIVERY SDH + where SDH.SDHNUM_0 = :sdhnum + """, + sdhnum=delivery + ).first()['CFMFLG_0'] + if result == 2: + return True + return False + +def process_file(edi_filename: pathlib.Path): + """ + Convert a specific EDI file into an import file. + """ + def fix_uom(uom): + x3_uom = '' + if uom == 'CA': + x3_uom = 'CS' + else: + x3_uom = uom + return x3_uom + + warehouse_receipt = Receipt() + pohnum = '' + for fields in tokens_from_edi_file(edi_filename): + if fields[0] == "W17": + _, _, rcpdat, _, sohnum, sdhnum = fields[:6] + warehouse_receipt.sdhnum = sdhnum + validated = check_shipment_status(sdhnum) + warehouse_receipt.header.rcpdat = datetime.datetime.strptime( + rcpdat, "%Y%m%d" + ).date() # 20230922 + if fields[0] == "N9" and fields[1] == "PO": + pohnum = fields[2] + if fields[0] == "W07": + # W07*1023*CA**PN*C08249*LT*07032026A***UK*10077652082491 + # N9*LI*1000 + _, qty_str, uom, _, _, itmref, _, lot = fields[:8] + subdetail = ReceiptSubDetail( + pcu=fix_uom(uom), + qtypcu=int(qty_str), + lot=lot + ) + if fields[0] == 'N9' and fields[1] == 'LI': + # N9*LI*1000 + #line = fields[2] #This line isn't the line number from X3, it needs to be looked up + line = find_shipment_line(warehouse_receipt.sdhnum, itmref) + warehouse_receipt.append( + ReceiptDetail( + sdhnum=warehouse_receipt.sdhnum, + poplin=int(line), + itmref=itmref, + uom=fix_uom(uom), + qtyuom=int(qty_str) + ), + subdetail, + ) + time_stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + #new_944_alert(sdhnum, pohnum, warehouse_receipt.header.rcpdat)#TODO is this needed? + # with yamamotoyama.x3_imports.open_import_file( + # IMPORTS_DIRECTORY / f"ZPTHI_{warehouse_receipt.sdhnum}_{time_stamp}.dat" + # ) as import_file: + # warehouse_receipt.output(import_file) + import_receipt(warehouse_receipt) + + +def import_receipt(warehouse_receipt): + """send the shipment information to the shandex staging database""" + with yamamotoyama.get_connection() as data_base: + result = data_base.query( + """ + SELECT + sdhnum + FROM [staging].[dbo].[shandex_receipts] + where sdhnum = :order + """, + order=warehouse_receipt.sdhnum, + ).all() + if not result: + with data_base.transaction(): + data_base.query( + INSERT_RECEIPT, + sdhnum=warehouse_receipt.sdhnum, + E='E', + prhfcy=warehouse_receipt.header.bpcord, + rcpdat=warehouse_receipt.header.rcpdat.strftime("%Y%m%d"), + pthnum='', + bpsnum=warehouse_receipt.header.stofcy, + cur=warehouse_receipt.header.cur, + star71=warehouse_receipt.header.star71, + star72=warehouse_receipt.header.star72, + star81=warehouse_receipt.header.star81, + star82=warehouse_receipt.header.star82 + ) + with data_base.transaction(): + for detail in warehouse_receipt.details: + detail.qtyuom = detail.check_subdetail_qty() + for subdetail in detail.subdetails: + data_base.query( + INSERT_RECEIPT_DETAILS, + L='L', + sdhnum=detail.sdhnum, + sddlin=detail.poplin, + itmref=detail.itmref, + uom=detail.uom, + qtyuom=detail.qtyuom, + pjt=detail.pjt, + star65=detail.star65, + star91=detail.star91, + star92=detail.star92, + S='S', + sta=subdetail.sta, + pcu=subdetail.pcu, + qtypcu=subdetail.qtypcu, + lot=subdetail.lot, + bpslot=subdetail.bpslot, + sernum=subdetail.sernum + ) + else: + simple_email_notification.email_noticication(['bleeson@stashtea.com'],'Shandex Receipt Error',[f'{warehouse_receipt.sdhnum} already exists, what happened?']) + + +@dataclasses.dataclass +class ReceiptSubDetail: + """ + Information that goes onto a receipt sub-detail line, taken from ZPTHI template. + """ + + sta: str = "A" + pcu: str = "" + qtypcu: int = 0 + loc: str = "" + lot: str = "" + bpslot: str = "" + sernum: str = "" + + def append(self, receipt_subdetail): + self.qtypcu += receipt_subdetail.qtypcu + + def stojous(self, shipment, item) -> typing.List[str]: + """ + Convert grouped lot quantities into individual STOJOU records to fit on receipt + """ + with yamamotoyama.get_connection() as database: + details = ( + database.query( + """ + select + 'S' [Code], + 'A' [STA_0], + [STJ].[PCU_0], + cast(cast(-1*[STJ].[QTYSTU_0] as int) as nvarchar) [QTYPCU_0], + [STJ].[LOT_0], + '' [BPSLOT_0], + '' [SERNUM_0] + from [PROD].[STOJOU] [STJ] + where + [STJ].[VCRNUM_0] = :sdhnum + and [STJ].[ITMREF_0] = :itmref + and [STJ].[LOT_0] = :lot + and [STJ].[TRSTYP_0] = 4 + """, + sdhnum=shipment, + itmref=item, + lot=self.lot, + ) + .all() + ) + return details + + + def convert_to_strings(self) -> typing.List[str]: + """ + Convert to strings for X3 import writing. + """ + return yamamotoyama.x3_imports.convert_to_strings( + [ + "S", + self.sta, + self.pcu, + self.qtypcu, + self.lot, + self.bpslot, + self.sernum, + ] + ) + + +@dataclasses.dataclass +class ReceiptDetail: + """ + Information that goes on a receipt detail line, taken from ZPTHI template. + """ + + sdhnum: str = "" + poplin: int = 0 + itmref: str = "" + itmdes: str = "" + uom: str = "" + qtyuom: int = 0 + pjt: str = "" + star65: str = "" + star91: str = "" + star92: str = "" + subdetails: typing.List[ReceiptSubDetail] = dataclasses.field( + default_factory=list + ) + + def append(self, subdetail: ReceiptSubDetail): + """ + Add subdetail + """ + subdetail.pcu = self.uom + self.subdetails.append(subdetail) + + def check_subdetail_qty(self): + """ + Check for shortages by totaling up subdetail quantities. + """ + total_cases = 0 + for subdetail in self.subdetails: + total_cases -= subdetail.qtypcu + return abs(total_cases) + + def convert_to_strings(self) -> typing.List[str]: + """ + Convert to strings for X3 import writing. + """ + def fix_uom(uom): + x3_uom = '' + if uom == 'CA': + x3_uom = 'CS' + else: + x3_uom = uom + return x3_uom + self.qty = self.check_subdetail_qty() + return yamamotoyama.x3_imports.convert_to_strings( + [ + "L", + self.sdhnum, + self.poplin, + self.itmref, + fix_uom(self.uom), + self.qty, + self.star65, + self.star91, + self.star92, + ] + ) + + def __eq__(self, item: typing.Any) -> bool: + """ + Test for equality + """ + if isinstance(item, str): + return self.itmref == item + if isinstance(item, ReceiptDetail): + return self.itmref == item.itmref + return False + + # def fill(self):#not needed for receipts + # """ + # Set soplin & itmdes from itmref & sohnum + # """ + + # def get() -> records.Record: + # with yamamotoyama.get_connection() as database: + # how_many = ( + # database.query( + # """ + # select + # count(*) as [how_many] + # from [PROD].[SORDERP] as [SOP] + # where + # [SOP].[SOHNUM_0] = :sohnum + # and [SOP].[ITMREF_0] = :itmref + # """, + # sohnum=self.sohnum, + # itmref=self.itmref, + # ) + # .first() + # .how_many + # ) + # if how_many == 1: + # return database.query( + # """ + # select top 1 + # [SOP].[SOPLIN_0] + # ,[SOP].[ITMDES1_0] + # ,[SOP].[SAU_0] + # from [PROD].[SORDERP] as [SOP] + # where + # [SOP].[SOHNUM_0] = :sohnum + # and [SOP].[ITMREF_0] = :itmref + # order by + # [SOP].[SOPLIN_0] + # """, + # sohnum=self.sohnum, + # itmref=self.itmref, + # ).first() + + + result = get() + self.soplin = result.SOPLIN_0 + self.itmdes = result.ITMDES1_0 + self.sau = result.SAU_0 + + +@dataclasses.dataclass +class ReceiptHeader: + """ + Information that goes on a receipt header, taken from ZPTHI template. + """ + + stofcy: str = "" + bpcord: str = "" + prhfcy: str = "" + rcpdat: datetime.date = datetime.date(1753, 1, 1) + pthnum: str = "" + bpsnum: str = "" + cur: str = "USD" + star71 = "" + star72 = "" + star81 = "" + star82 = "" + + + def convert_to_strings(self) -> typing.List[str]: + """ + Convert to X3 import line + """ + return yamamotoyama.x3_imports.convert_to_strings( + [ + "E", + self.bpcord, + self.rcpdat.strftime("%Y%m%d"), + self.pthnum, + self.stofcy, + self.cur, + self.star71, + self.star72, + self.star81, + self.star82, + ] + ) + + +class ReceiptDetailList: + """ + List of receipt details + """ + + _details: typing.List[ReceiptDetail] + _item_set: typing.Set[str] + + def __init__(self): + self._details = [] + self._item_set = set() + + def append( + self, + receipt_detail: ReceiptDetail, + receipt_subdetail: ReceiptSubDetail, + ): + """ + Append + """ + itmref = receipt_detail.itmref + if itmref in self._item_set: + for detail in self._details: + for subdetail in detail.subdetails: + if detail.itmref == itmref and subdetail.lot == receipt_subdetail.lot: + subdetail.append(receipt_subdetail) + return + if detail.itmref == itmref: + detail.subdetails.append(receipt_subdetail) + return + self._item_set.add(itmref) + #receipt_detail.fill() + receipt_detail.append(receipt_subdetail) + self._details.append(receipt_detail) + + def __iter__(self): + return iter(self._details) + + +class Receipt: + """ + Warehouse receipt, both header & details + """ + + header: ReceiptHeader + details: ReceiptDetailList + _sdhnum: str + + def __init__(self): + self.header = ReceiptHeader() + self._sdhnum = "" + self.details = ReceiptDetailList() + + def append( + self, + receipt_detail: ReceiptDetail, + receipt_subdetail: ReceiptSubDetail, + ): + """ + Add detail information. + """ + self.details.append(receipt_detail, receipt_subdetail) + + @property + def sdhnum(self): + """ + shipment number + """ + return self._sdhnum + + @sdhnum.setter + def sdhnum(self, value: str): + if self._sdhnum != value: + self._sdhnum = value + if value: + self._fill_info_from_shipment() + + def _get_shipment_from_x3(self) -> records.Record: + """ + Fetch shipment from X3 database. + """ + with yamamotoyama.get_connection() as db_connection: + return db_connection.query( + """ + select + [SDH].[STOFCY_0], + [SDH].[SDHNUM_0], + [SDH].[SALFCY_0], + [SDH].[BPCORD_0], + [SDH].[CUR_0], + [SDH].[SOHNUM_0] + from [PROD].[SDELIVERY] [SDH] + where + [SDH].[SDHNUM_0] = :shipment + """, + shipment=self.sdhnum, + ).first() + + def _fill_info_from_shipment(self): + """ + When we learn the SOHNUM, we can copy information from the sales order. + """ + result = self._get_shipment_from_x3() + self.header.stofcy = result.STOFCY_0 + self.header.sdhnum = result.SDHNUM_0 + self.header.salfcy = result.SALFCY_0 + self.header.bpcord = result.BPCORD_0 + self.header.cur = result.CUR_0 + self.header.sohnum = result.SOHNUM_0 + + + def output(self, import_file: typing.TextIO): + """ + Output entire order to import_file. + """ + output = functools.partial( + yamamotoyama.x3_imports.output_with_file, import_file + ) + output(self.header.convert_to_strings()) + for detail in self.details: + output(detail.convert_to_strings()) + for subdetail in detail.subdetails: + shipment = detail.sdhnum + item = detail.itmref + for record in subdetail.stojous(shipment, item): + record_list = [ + record['Code'], + record['STA_0'], + record['PCU_0'], + record['QTYPCU_0'], + record['LOT_0'], + record['BPSLOT_0'], + record['SERNUM_0'] + ] + #pprint.pprint(record_list) + output(record_list) + + +if __name__ == "__main__": + main() diff --git a/edi_947.py b/edi_947.py index b9cff5a..f49a005 100644 --- a/edi_947.py +++ b/edi_947.py @@ -24,11 +24,22 @@ import pprint from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText +import os +import base64 +import google.auth +import pickle +# Gmail API utils +from googleapiclient.discovery import build +from google_auth_oauthlib.flow import InstalledAppFlow +from google.auth.transport.requests import Request + import records # type: ignore import yamamotoyama # type: ignore import yamamotoyama.x3_imports # type: ignore +SCOPES = ['https://mail.google.com/'] + THIS_DIRECTORY = pathlib.Path(__file__).parent X12_DIRECTORY = THIS_DIRECTORY / "incoming" IMPORTS_DIRECTORY = THIS_DIRECTORY / "x3_imports" @@ -81,6 +92,38 @@ def main(): combine_zscs() +def gmail_authenticate(): + creds = None + # the file token.pickle stores the user's access and refresh tokens, and is + # created automatically when the authorization flow completes for the first time + if os.path.exists("token.pickle"): + with open("token.pickle", "rb") as token: + creds = pickle.load(token) + # if there are no (valid) credentials availablle, let the user log in. + if not creds or not creds.valid: + if creds and creds.expired and creds.refresh_token: + creds.refresh(Request()) + else: + flow = InstalledAppFlow.from_client_secrets_file('credentials.json', SCOPES) + creds = flow.run_local_server(port=0) + # save the credentials for the next run + with open("token.pickle", "wb") as token: + pickle.dump(creds, token) + return build('gmail', 'v1', credentials=creds) + + +def gmail_send_message(service, payload): + create_message = {"raw": payload} + # pylint: disable=E1101 + send_message = ( + service.users() + .messages() + .send(userId="me", body=create_message) + .execute() + ) + return send_message + + def combine_zscs(): """ Collect all ZSCS imports into a single file for easy import. @@ -152,13 +195,16 @@ def stock_movement_alert(itmref, qty, lot, status): msg['Subject'] = 'New Stock Change from Shandex' msg['Precedence'] = 'bulk' msg['From'] = 'x3report@stashtea.com' - msg['To'] = 'isenn@yamamotoyama.com, vgomez@yamamotoyama.com' - msg['CC'] = 'bleeson@stashtea.com' + msg['To'] = 'woninventory@stashtea.com' + msg['CC'] = 'bleeson@stashtea.com,icarrera@yamamotoyama.com' emailtext = f'Item: {itmref}\nQty: {qty}\nLot: {lot}\nStatus: {DAMAGE_CODE_MAPPING[status]}\nReason: {DAMAGE_CODE_DESCRIPTIONS_MAPPING[status]}' msg.attach(MIMEText(emailtext, 'plain')) - with smtplib.SMTP_SSL("smtp.gmail.com", 465) as smtp: - smtp.login(user='x3reportmk2@yamamotoyama.com', password=r'n