天天看點
傳回

我心中的核心元件（可插拔的AOP）~分布式檔案上傳元件~基于FastDFS

回到目錄

一些概念

在大叔架構裡總覺得缺點什麼，在最近的項目開發中，終于知道缺什麼了，分布式檔案存儲元件，就是缺它，呵呵，對于分布式檔案存儲來說，業界比較公認的是FastDFS元件，它自己本身就是叢集機制，有自己的路由選擇和檔案存儲兩個部分，我們通過FastDFS的用戶端進行上傳後，它會傳回一個在FastDFS上存儲的路徑，這當然是IO路徑，我們隻要在伺服器上開個Http伺服器，就可以以Http的方法通路你的檔案了。

我的元件實作方式

前端上傳控件（表單方式，swf方式，js方法均可）将檔案流傳給我們的FastDFS用戶端，通過用戶端與服務端建立Socket連接配接，将資料包發給FastDFS服務端并等待傳回，上傳成功後傳回路徑，我們可以對路徑進行ＨＴＴＰ的處理，并存入資料庫

fastDFS配合nginx伺服器自動生成指定尺寸的圖像

原圖像位址： http://www.fastdfs.com/demo/pictruename.jpg

指定尺寸的圖像位址：http://www.fastdfs.com/demo/pictruename_100x100.jpg

技術實作

１　一個接口，定義三種上傳規格，普通檔案，圖像檔案和視訊檔案（一般需要對它進行截力）

public interface IFileUploader
    {
        /// <summary>
        /// 上傳視訊檔案
        /// </summary>
        /// <param name="param"></param>
        /// <returns></returns>
        VideoUploadResult UploadVideo(VideoUploadParameter param);
        /// <summary>
        /// 上傳普通檔案
        /// </summary>
        /// <param name="filePath"></param>
        /// <returns></returns>
        FileUploadResult UploadFile(FileUploadParameter param);
        /// <summary>
        /// 上傳圖檔
        /// </summary>
        /// <param name="param"></param>
        /// <returns></returns>
        /// <remarks>Update:cyr(Ben) 20150317</remarks>
        ImageUploadResult UploadImage(ImageUploadParameter param);
    }

２　一批方法參數，包括了檔案，圖像和視訊等

/// <summary>
    /// 檔案上傳參數基類
    /// </summary>
    public abstract class UploadParameterBase
    {
        public UploadParameterBase()
        {
            MaxSize = 1024 * 1024 * 8;
        }
        /// <summary>
        /// 前一次上傳時生成的伺服器端檔案名，如果需要斷點續傳，需傳入此檔案名
        /// </summary>
        public string ServiceFileName { get; set; }
        /// <summary>
        /// 檔案流
        /// </summary>
        public Stream Stream { get; set; }
        /// <summary>
        /// 檔案名
        /// </summary>
        public string FileName { get; set; }
        /// <summary>
        /// 檔案大小限制（機關bit 預設1M）
        /// </summary>
        public int MaxSize
        {
            get;
            protected set;
        }
        /// <summary>
        /// 上傳檔案類型限制
        /// </summary>
        public string[] FilenameExtension
        {
            get;
            set;
        }

    }      
/// <summary>
    /// 圖檔上傳參數對象
    /// </summary>
    public class ImageUploadParameter : UploadParameterBase
    {
        /// <summary>
        /// 構造方法
        /// </summary>
        /// <param name="stream"></param>
        /// <param name="fileName"></param>
        /// <param name="filenameExtension">預設支援常用圖檔格式</param>
        /// <param name="maxSize"></param>
        public ImageUploadParameter(Stream stream, string fileName, string[] filenameExtension = null, int maxSize = 3)
        {
            base.Stream = stream;
            base.FileName = fileName;
            base.MaxSize = maxSize;
            base.FilenameExtension = filenameExtension ?? new string[] { ".jpeg", ".jpg", ".gif", ".png" }; ;

        }
        /// <summary>
        /// 構造方法
        /// </summary>
        /// <param name="stream"></param>
        /// <param name="fileName"></param>
        /// <param name="maxSize">機關為M</param>
        public ImageUploadParameter(Stream stream, string fileName, int maxSize)
            : this(stream, fileName, null, maxSize) { }

    }

３　一批傳回類型，包括對檔案，圖像和視訊等方法的傳回資料的定義

　　/// <summary>
    /// 上傳檔案傳回對象基類
    /// </summary>
    public abstract class UploadResultBase
    {
        /// <summary>
        /// 傳回檔案位址
        /// </summary>
        public string FilePath { get; set; }
        /// <summary>
        /// 錯誤消息清單
        /// </summary>
        public string ErrorMessage { get; set; }
        /// <summary>
        /// 是否上傳成功
        /// </summary>
        public bool IsValid { get { return string.IsNullOrWhiteSpace(ErrorMessage); } }
    }      
/// <summary>
    /// 視訊上傳傳回對象
    /// </summary>
    public class VideoUploadResult : UploadResultBase
    {
        /// <summary>
        /// 上傳的視訊截圖位址
        /// </summary>
        public List<string> ScreenshotPaths { get; set; }
        /// <summary> 
        /// 上傳狀态
        /// </summary>
        public UploadStatus UploadStatus { get; set; }

        public VideoUploadResult()
        {
            ScreenshotPaths = new List<string>();
        }
        /// <summary>
        /// 把VideoPath和ScreenshotPaths拼起來  以豎線（|）隔開
        /// </summary>
        /// <returns></returns>
        public override string ToString()
        {
            StringBuilder sb = new StringBuilder();
            sb.Append(FilePath);
            foreach (var item in ScreenshotPaths)
            {
                sb.Append("|" + item);
            }
            return sb.ToString();
        }
    }

４　一個使用FastDFS實作的檔案上傳實作類

/// <summary>
    /// 使用fastDFS完成檔案上傳
    /// </summary>
    internal class FastDFSUploader : IFileUploader
    {
        /// <summary>
        /// 目錄名,需要提前在fastDFS上建立
        /// </summary>
        public string DFSGroupName { get { return "tsingda"; } }
        /// <summary>
        /// FastDFS結點
        /// </summary>
        public StorageNode Node { get; private set; }
        /// <summary>
        /// 伺服器位址
        /// </summary>
        public string Host { get; private set; }
        /// <summary>
        /// 失敗次數
        /// </summary>
        protected int FaildCount { get; set; }

        public int MaxFaildCount { get; set; }

        public FastDFSUploader()
        {
            InitStorageNode();
            MaxFaildCount = 3;
        }

        #region Private Methods
        private void InitStorageNode()
        {
            Node = FastDFSClient.GetStorageNode(DFSGroupName);
            Host = Node.EndPoint.Address.ToString();
        }

        private List<string> CreateImagePath(string fileName)
        {
            List<string> pathList = new List<string>();
            string snapshotPath = "";
            //視訊截圖
            List<string> localList = new VideoSnapshoter().GetVideoSnapshots(fileName, out snapshotPath);
            foreach (var item in localList)
            {
                string aImage = SmallFileUpload(item);
                pathList.Add(aImage);
            }
            //清除本地多餘的圖檔，有的視訊截取的圖檔多，有的視訊截取的圖檔少
            string[] strArr = Directory.GetFiles(snapshotPath);
            try
            {
                foreach (var strpath in strArr)
                {
                    File.Delete(strpath);
                }
                Directory.Delete(snapshotPath);
            }
            catch (Exception ex)
            {
                Logger.Core.LoggerFactory.Instance.Logger_Info("删除圖檔截圖異常" + ex.Message);
            }
            return pathList;
        }
        private string SmallFileUpload(string filePath)
        {
            if (string.IsNullOrEmpty(filePath))
                throw new ArgumentNullException("filePath 參數不能為空");
            if (!File.Exists(filePath))
                throw new Exception("上傳的檔案不存在");
            byte[] content;
            using (FileStream streamUpload = new FileStream(filePath, FileMode.Open, FileAccess.Read))
            {
                using (BinaryReader reader = new BinaryReader(streamUpload))
                {
                    content = reader.ReadBytes((int)streamUpload.Length);
                }
            }
            string shortName = FastDFSClient.UploadFile(Node, content, "png");
            return GetFormatUrl(shortName);
        }
        /// <summary>
        /// 檔案分塊上傳，适合大檔案
        /// </summary>
        /// <param name="file"></param>
        /// <returns></returns>
        private string MultipartUpload(UploadParameterBase param)
        {
            Stream stream = param.Stream;
            if (stream == null)
                throw new ArgumentNullException("stream參數不能為空");
            int size = 1024 * 1024;
            byte[] content = new byte[size];
            Stream streamUpload = stream;
            //  第一個資料包上傳或擷取已上傳的位置
            string ext = param.FileName.Substring(param.FileName.LastIndexOf('.') + 1);
            streamUpload.Read(content, 0, size);
            string shortName = FastDFSClient.UploadAppenderFile(Node, content, ext);

            BeginUploadPart(stream, shortName);

            return CompleteUpload(stream, shortName);
        }
        /// <summary>
        /// 斷點續傳
        /// </summary>
        /// <param name="stream"></param>
        /// <param name="serverShortName"></param>
        private void ContinueUploadPart(Stream stream, string serverShortName)
        {
            var serviceFile = FastDFSClient.GetFileInfo(Node, serverShortName);
            stream.Seek(serviceFile.FileSize, SeekOrigin.Begin);
            BeginUploadPart(stream, serverShortName);
        }
        /// <summary>
        /// 從指定位置開始上傳檔案
        /// </summary>
        /// <param name="stream"></param>
        /// <param name="beginOffset"></param>
        /// <param name="serverShortName"></param>
        private void BeginUploadPart(Stream stream, string serverShortName)
        {
            try
            {
                int size = 1024 * 1024;
                byte[] content = new byte[size];

                while (stream.Position < stream.Length)
                {
                    stream.Read(content, 0, size);

                    var result = FastDFSClient.AppendFile(DFSGroupName, serverShortName, content);
                    if (result.Length == 0)
                    {
                        FaildCount = 0;
                        continue;
                    }
                }
            }
            catch (Exception ex)
            {
                Logger.Core.LoggerFactory.Instance.Logger_Info("上傳檔案中斷！" + ex.Message);
                if (NetCheck())
                {
                    //重試
                    if (FaildCount < MaxFaildCount)
                    {
                        FaildCount++;
                        InitStorageNode();
                        ContinueUploadPart(stream, serverShortName);
                    }
                    else
                    {
                        Logger.Core.LoggerFactory.Instance.Logger_Info("已達到失敗重試次數仍沒有上傳成功"); ;
                        throw ex;
                    }
                }
                else
                {
                    Logger.Core.LoggerFactory.Instance.Logger_Info("目前網絡不可用");
                    throw ex;
                }
            }
        }
        /// <summary>
        /// 網絡可用為True,否則為False
        /// </summary>
        /// <returns></returns>
        private bool NetCheck()
        {
            return NetworkInterface.GetIsNetworkAvailable();
        }
        /// <summary>
        /// 拼接Url
        /// </summary>
        /// <param name="shortName"></param>
        /// <returns></returns>
        private string GetFormatUrl(string shortName)
        {
            return string.Format("http://{0}/{1}/{2}", Host, DFSGroupName, shortName);
        }

        private string CompleteUpload(Stream stream, string shortName)
        {
            stream.Close();
            return GetFormatUrl(shortName);
        }

        private string GetShortNameFromUrl(string url)
        {
            if (string.IsNullOrEmpty(url))
                return string.Empty;
            Uri uri = new Uri(url);
            string urlFirstPart = string.Format("http://{0}/{1}/", Host, DFSGroupName);
            if (!url.StartsWith(urlFirstPart))
                return string.Empty;
            return url.Substring(urlFirstPart.Length);
        }
        #endregion

        #region IFileUploader 成員

        /// <summary>
        /// 上傳視訊
        /// </summary>
        /// <param name="param"></param>
        /// <returns></returns>
        public VideoUploadResult UploadVideo(VideoUploadParameter param)
        {
            VideoUploadResult result = new VideoUploadResult();
            string fileName = MultipartUpload(param);
            if (param.IsScreenshot)
            {
                result.ScreenshotPaths = CreateImagePath(fileName);
            }
            result.FilePath = fileName;
            return result;
        }

        /// <summary>
        /// 上傳普通檔案
        /// </summary>
        /// <param name="param"></param>
        /// <returns></returns>
        public FileUploadResult UploadFile(FileUploadParameter param)
        {
            var result = new FileUploadResult();
            try
            {
                string fileName = MultipartUpload(param);
                result.FilePath = fileName;
            }
            catch (Exception ex)
            {

                result.ErrorMessage = ex.Message;
            }
            return result;
        }

        /// <summary>
        /// 上傳圖檔
        /// </summary>
        /// <param name="param"></param>
        /// <param name="message"></param>
        /// <returns></returns>
        public ImageUploadResult UploadImage(ImageUploadParameter param)
        {
            byte[] content;
            string shortName = "";
            string ext = System.IO.Path.GetExtension(param.FileName).ToLower();
            if (param.FilenameExtension != null && param.FilenameExtension.Contains(ext))
            {
                if (param.Stream.Length > param.MaxSize)
                {
                    return new ImageUploadResult
                    {
                        ErrorMessage = "圖檔大小超過指定大小" + param.MaxSize / 1048576 + "M，請重新選擇",
                        FilePath = shortName
                    };
                }
                else
                {
                    using (BinaryReader reader = new BinaryReader(param.Stream))
                    {
                        content = reader.ReadBytes((int)param.Stream.Length);
                    }

                    shortName = FastDFSClient.UploadFile(Node, content, ext.Contains('.') ? ext.Substring(1) : ext);
                }
            }
            else
            {
                return new ImageUploadResult
                {
                    ErrorMessage = "檔案類型不比對",
                    FilePath = shortName
                };

            }
            return new ImageUploadResult
            {
                FilePath = CompleteUpload(param.Stream, shortName),
            };
        }
        #endregion
    }

５　一個檔案上傳的生産者，經典的單例模式的展現

/// <summary>
    /// 檔案上傳生産者
    /// </summary>
    public class FileUploaderFactory
    {
        /// <summary>
        /// 上傳執行個體
        /// </summary>
        public readonly static IFileUploader Instance;
        private static object lockObj = new object();
        static FileUploaderFactory()
        {
            if (Instance == null)
            {
                lock (lockObj)
                {
                    Instance = new FastDFSUploader();
                }
            }
        }
    }

６　前台的檔案上傳控件，你可以随便選擇了，它與前台是解耦的，沒有什麼關系，用哪種方法實作都可以，呵呵

　　　　[HttpPost]
        public ActionResult Index(FormCollection form)
        {
            var file = Request.Files[0];
            var result = Project.FileUpload.FileUploaderFactory.Instance.UploadFile(new Project.FileUpload.Parameters.FileUploadParameter
            {
                FileName = file.FileName,
                Stream = file.InputStream,
            });
            ViewBag.Path = result.FilePath;
            return View();
        }

最後我們看一下我的Project.FileUpload的完整結構

它隸屬于大叔的Project.Frameworks集合，我們在這裡，對Project.FileUpload說一聲：Hello FileUpload,We wait for you for a long time...

 回到目錄

作者：倉儲大叔，張占嶺，

榮譽：微軟MVP

QQ：853066980

支付寶掃一掃，為大叔打賞!

我心中的核心元件（可插拔的AOP）~分布式檔案上傳元件~基于FastDFS
上一篇: 阿裡雲ECS雲伺服器通用型g6和g5的差別以及選擇建議
下一篇: 小猿日記 - 程式猿的日常日記（3）口水記小結不正經語錄聲明公衆号