Move Files from FTP Server to AWS S3 bucket

PHOTO EMBED

Tue Oct 11 2022 08:14:11 GMT+0000 (Coordinated Universal Time)

Saved by @lscode #python

import ftplib
import subprocess

# Credentials
username = ""
password = ""

# FTP Server Info
ip = ""
dnsName = ""
port = 

# QUA - FTP Location of Files
ftppath = "/BI_QUA/Expedicao"

# QUA - Bucket S3 Location of Files
bucketPath = "s3://analytics-qa-data-raw/expedicoes/"


#Open FTP Connection
def openFTPServer(ip, port, username, password):
    server = ftplib.FTP()
    server.connect(ip, port)
    server.encoding = "utf-8"
    server.login(username, password)
    return server

#Return list of files available on source
def getListOfFiles(server, path):
    files_list = []
    if server is not None:
        server.cwd(path)
        server.dir(files_list.append)
    return files_list

#Clean some string Junk and returns clean filename
def getNamesOfFiles(files_list):
    filenames = []
    for line in files_list:
        elem = line.split(" ")
        filenames.append(elem[-1])

    return filenames

#Download Files
def getFiles(server, filenames):
    for namefile in filenames:
        with open(namefile, "wb") as file:
            server.retrbinary(f"RETR {namefile}", file.write)


#Ends a FTP connection
def closeFTPConnection(server):
    server.quit()


#Moves the files to the bucket
def moveFilesToBucket(files, bucketPath):
    for file in files:
        subprocess.call(['aws', 's3', 'mv', file, bucketPath])


# MAIN Execution
server = openFTPServer(ip, port, username, password)
filesList = getListOfFiles(server, ftppath)
filenames = getNamesOfFiles(filesList)
getFiles(server, filenames)
closeFTPConnection(server)
moveFilesToBucket(filenames,bucketPath)
content_copyCOPY

This code is used to move some files, from a FTP Server, for a S3 bucket, using a tunnel Site to Site VPN.