Move Files from FTP Server to AWS S3 bucket
Tue Oct 11 2022 08:14:11 GMT+0000 (UTC)
Saved by
@lscode
#python
import ftplib
import subprocess
# Credentials
username = ""
password = ""
# FTP Server Info
ip = ""
dnsName = ""
port =
# QUA - FTP Location of Files
ftppath = "/BI_QUA/Expedicao"
# QUA - Bucket S3 Location of Files
bucketPath = "s3://analytics-qa-data-raw/expedicoes/"
#Open FTP Connection
def openFTPServer(ip, port, username, password):
server = ftplib.FTP()
server.connect(ip, port)
server.encoding = "utf-8"
server.login(username, password)
return server
#Return list of files available on source
def getListOfFiles(server, path):
files_list = []
if server is not None:
server.cwd(path)
server.dir(files_list.append)
return files_list
#Clean some string Junk and returns clean filename
def getNamesOfFiles(files_list):
filenames = []
for line in files_list:
elem = line.split(" ")
filenames.append(elem[-1])
return filenames
#Download Files
def getFiles(server, filenames):
for namefile in filenames:
with open(namefile, "wb") as file:
server.retrbinary(f"RETR {namefile}", file.write)
#Ends a FTP connection
def closeFTPConnection(server):
server.quit()
#Moves the files to the bucket
def moveFilesToBucket(files, bucketPath):
for file in files:
subprocess.call(['aws', 's3', 'mv', file, bucketPath])
# MAIN Execution
server = openFTPServer(ip, port, username, password)
filesList = getListOfFiles(server, ftppath)
filenames = getNamesOfFiles(filesList)
getFiles(server, filenames)
closeFTPConnection(server)
moveFilesToBucket(filenames,bucketPath)
content_copyCOPY
This code is used to move some files, from a FTP Server, for a S3 bucket, using a tunnel Site to Site VPN.
Comments