Simple python application to fetch logs from the FTP server and ship it to your application server.

When I joined, I was given a brief introduction about how product works and normal engineering and infrastructure.

After a couple of days, I was assigned a task of creating an application which will continuously monitor files in one remote FTP location and if any file/files come up in that location my application should figure out and download only those new file/files in the server, parse the logs from that file to my application server and delete that downloaded file from the server and again python application should wait for next file to come in. This should be created as a single docker image.

So the idea was to run a docker container with all these features.

I wrote python and YAML files for testing my python containerized application

python file - for communicating with FTP server

YAML file - for reading confidential information, such as username and password

So here my application runs on 514 port and ready to receive data. so I have created the FTP server for testing (because I want to read files from FTP location)

This is my main file which connects to the FTP server and monitors files.

main.py

#!/usr/bin/python3
import os
import time
import ftplib
import yaml
import socket
import sys
import socket
import logging
#time=str(os.system("date +%Y%m%d%H%M%S"))
stream = open('variables.yaml', 'r')
data = yaml.safe_load(stream)
#print (data)
ftplogging=data['loggingfile']
user = data['user']
passwd = data['passwd']
host = data['host']
PORT = data["port"]
#latest_time=None
#latest_name=None
filelocation = data["filelocation"]
logging.basicConfig(filename=ftplogging,level=logging.DEBUG)
s = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
try:
  s.connect((host, PORT))
  class Connector:
###Downloading new file from ftp if found
    def ftpconn(self):
      while True:
        #listing=[]
        ftp = ftplib.FTP(host)
        ftp.login(user=user,passwd=passwd)
        ftp.cwd(filelocation)
        #ftp.retrlines("LIST", listing.append)
        #listing.append(ftp.nlst())
        #print (len(ftp.nlst()))
        for i in range(0,len(ftp.nlst())):
          f = ftp.nlst()[i]
          #print (f)
          obj = open("file.log","r+")
          content = obj.read()
          time=ftp.voidcmd("MDTM "+ftp.nlst()[i])
          if ftp.nlst()[i] in content:
            logging.info("file has already been downloaded")
            pass
          else:
            logging.info("Downloading file",ftp.nlst()[i])
            with open(ftp.nlst()[i],'wb') as file:
              ftp.retrbinary('RETR %s' %f ,file.write)
              logging.info("Download completed")
              #ftp.quit()
              logging.debug('New file has been updated in FTP directory, downloading it...')
              logging.info('New file has been updated in FTP directory, downloading it....')
###Sending data through socket on 514 port
              obj.write(ftp.nlst()[i]+'\n')
              logging.info("[+] Connected with Server")
              obj2=open(ftp.nlst()[i],"r")
              for k in obj2.readlines():
                print (k)
                #s.send(str(k.encode('ascii')))
                s.send(k.encode('utf8'))
                time.sleep(0.5)
              logging.info("Sent logs to Application!")
              obj.close()
              os.remove(ftp.nlst()[i])
      s.close()
  con=Connector()
  con.ftpconn()
except ConnectionRefusedError:
  print ("Your Application is not listening on 514 port")
  exit(0)

I have created a variables file, so my main python file is reading data from variables.yaml. I can't pass credentials directly into my python script.

host: 192.168.86.68
user: "ftpuser"
passwd: "ftpuser"
filelocation: "//files/"
port: 514
loggingfile: "ftplogging"

Assuming I am sending data through some application which runs on UDP port 514

senddata.py

import socket
import yaml
import sys
import time
from ftp.py import ftpconn
stream = open('variables.yaml', 'r')
data = yaml.safe_load(stream)
HOST = data["host"]
PORT = data["port"]

s = socket.socket(socket.AF_INET,   socket.SOCK_STREAM)
s.connect((HOST, PORT))
print("[+] Connected with Server")
file = open(obj,"r")
for i in file.readlines():
  s.send(i.encode())
  time.sleep(0.5)
  
s.close()

Assuming my application will run on 514 port and ready to accept data. For testing, I have written a sample code to make sure that data is being ingested into my application.

receivedata.py

import socket
import sys


HOST = "192.168.86.68"
PORT = 514

s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((HOST, PORT))
s.listen(5)
s.settimeout(172800)
print("Listening ...")

while True:
    conn, addr = s.accept()
    print("[+] Client connected: ", addr)

    # get file name to download
    #f = open("recieved_logfile", "wb")
    while True:
        # get file bytes
        data = conn.recv(4096)
        print (data.decode('utf8'))
        if not data:
            break
        # write bytes on file
   #     f.write(data)
#    f.close()
    print("[+] Download complete!")

    # close connection
#    conn.close()
    print("[-] Client disconnected")
    sys.exit(0)