here.<\/a>.<\/p>\nTo delete the files from FTP or clean a certain catalog, you may use this C# code:<\/p>\n
using System.IO;\r\nusing System.Collections.Generic;\r\nusing Microsoft.Azure.Management.DataFactories.Models;\r\nusing Microsoft.Azure.Management.DataFactories.Runtime;\r\nusing System.Net;\r\nusing System.Linq;\r\n\r\nnamespace DeleteFromFTP\r\n{\r\n public class DeleteFromFTP : IDotNetActivity\r\n {\r\n public IDictionary<string, string> Execute(\r\n IEnumerable linkedServices,\r\n IEnumerable datasets,\r\n Activity activity,\r\n IActivityLogger logger)\r\n {\r\n\t\t\t\/\/Creating refrences for the pipeline\r\n DotNetActivity dotNetActivity = (DotNetActivity)activity.TypeProperties;\r\n string Path = dotNetActivity.ExtendedProperties[\"Path\"];\r\n string IP = dotNetActivity.ExtendedProperties[\"IP\"];\r\n string Login = dotNetActivity.ExtendedProperties[\"Login\"];\r\n string Password = dotNetActivity.ExtendedProperties[\"Password\"];\r\n AzureStorageLinkedService linkedService = linkedServices.First(ls => ls.Name == dotNetActivity.PackageLinkedService).Properties.TypeProperties as AzureStorageLinkedService;\r\n\r\n DeleteFTPDirectory(Path+\"\/\", IP, Login, Password);\r\n\r\n \r\n return new Dictionary<string, string>();\r\n }\r\n public static List DirectoryListing(string Path, string ServerAdress, string Login, string Password)\r\n {\r\n FtpWebRequest request = (FtpWebRequest)WebRequest.Create(\"ftp:\/\/\" + ServerAdress + Path);\r\n request.Credentials = new NetworkCredential(Login, Password);\r\n\r\n request.Method = WebRequestMethods.Ftp.ListDirectory;\r\n\r\n FtpWebResponse response = (FtpWebResponse)request.GetResponse();\r\n Stream responseStream = response.GetResponseStream();\r\n StreamReader reader = new StreamReader(responseStream);\r\n\r\n List result = new List();\r\n\r\n while (!reader.EndOfStream)\r\n {\r\n result.Add(reader.ReadLine());\r\n }\r\n\r\n reader.Close();\r\n response.Close();\r\n\r\n return result;\r\n }\r\n public static void DeleteFTPFile(string Path, string ServerAdress, string Login, string Password)\r\n {\r\n FtpWebRequest clsRequest = (System.Net.FtpWebRequest)WebRequest.Create(\"ftp:\/\/\" + ServerAdress + Path);\r\n clsRequest.Credentials = new System.Net.NetworkCredential(Login, Password);\r\n\r\n clsRequest.Method = WebRequestMethods.Ftp.DeleteFile;\r\n\r\n string result = string.Empty;\r\n FtpWebResponse response = (FtpWebResponse)clsRequest.GetResponse();\r\n long size = response.ContentLength;\r\n Stream datastream = response.GetResponseStream();\r\n StreamReader sr = new StreamReader(datastream);\r\n result = sr.ReadToEnd();\r\n sr.Close();\r\n datastream.Close();\r\n response.Close();\r\n }\r\n public static void DeleteFTPDirectory(string Path, string ServerAdress, string Login, string Password)\r\n {\r\n FtpWebRequest clsRequest = (System.Net.FtpWebRequest)WebRequest.Create(\"ftp:\/\/\" + ServerAdress + Path);\r\n clsRequest.Credentials = new System.Net.NetworkCredential(Login, Password);\r\n\r\n List filesList = DirectoryListing(Path, ServerAdress, Login, Password);\r\n\r\n foreach (string file in filesList)\r\n {\r\n DeleteFTPFile(Path + file, ServerAdress, Login, Password);\r\n }\r\n\r\n\r\n }\r\n }\r\n}<\/pre>\nAfter compiling the code and uploading it to blob, the pipeline activity will look like this:<\/p>\n
{\r\n \"name\": \"Delete_from_FTP\",\r\n \"properties\": {\r\n \"description\": \"Activity to delete files from a folder\",\r\n \"activities\": [\r\n {\r\n \"type\": \"DotNetActivity\",\r\n \"typeProperties\": {\r\n \"assemblyName\": \"DeleteFromFTP.dll\",\r\n \"entryPoint\": \"DeleteFromFTP.DeleteFromFTP\",\r\n \"packageLinkedService\": ,\r\n \"packageFile\": ,\r\n \"extendedProperties\": {\r\n \"Path\": ,\r\n\t\t\t\t\t\t\"IP\": ,\r\n\t\t\t\t\t\t\"Login\": ,\r\n\t\t\t\t\t\t\"Password\": ,\r\n }\r\n },\r\n \"inputs\": [\r\n {\r\n \"name\": \"Input Dataset\r\n }\r\n ],\r\n \"outputs\": [\r\n {\r\n \"name\": \"Output Dataset\"\r\n }\r\n ],\r\n \"policy\": {\r\n \"timeout\": \"01:00:00\",\r\n \"concurrency\": 1,\r\n \"longRetry\": 10,\r\n \"longRetryInterval\": \"01:00:00\"\r\n },\r\n \"scheduler\": {\r\n \"frequency\": \"Day\",\r\n \"interval\": 1\r\n },\r\n \"name\": \"Delete_from_FTP\",\r\n \"linkedServiceName\": \"AzureBatchLinkedService\"\r\n }\r\n ],\r\n \r\n \"isPaused\": false,\r\n \"hubName\": \"Your_hub_name\",\r\n \"pipelineMode\": \"Scheduled\"\r\n }\r\n}\r\n<\/pre>\nIf you don’t want to include the login and the password in the activity, you will have to do so in C# code.<\/p>\n
A simple and tested code \ud83d\ude42<\/p>\n","protected":false},"excerpt":{"rendered":"
During development of an ETL process in Azure Data Factory, you may come across a need to delete the data from FTP. It may be neccessary e.g. when you want to make place for new incoming data or you don’t want your pipeline to pull the old data twice. The solution of the problem lays in a custom activity.<\/p>\n
Read More<\/a><\/div>","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[28],"tags":[16],"class_list":["post-98","post","type-post","status-publish","format-standard","hentry","category-azure","tag-azure"],"_links":{"self":[{"href":"https:\/\/cwiok.pl\/index.php\/wp-json\/wp\/v2\/posts\/98","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/cwiok.pl\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/cwiok.pl\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/cwiok.pl\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/cwiok.pl\/index.php\/wp-json\/wp\/v2\/comments?post=98"}],"version-history":[{"count":0,"href":"https:\/\/cwiok.pl\/index.php\/wp-json\/wp\/v2\/posts\/98\/revisions"}],"wp:attachment":[{"href":"https:\/\/cwiok.pl\/index.php\/wp-json\/wp\/v2\/media?parent=98"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/cwiok.pl\/index.php\/wp-json\/wp\/v2\/categories?post=98"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/cwiok.pl\/index.php\/wp-json\/wp\/v2\/tags?post=98"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}