天天看点
返回

nginx upload模块+python 后端处理模仿fastdfs实现文件存取

前提条件：

1.编译安装带有nginx upload 模块的nginx，安装脚本如下。

2.配置nginx的conf文件，一个server配置upload上传，一个server配置download下载

3.使用django快速部署python后端服务，对nginx存入的文件进行处理，并返回下载的url

注释：如果不适用后端处理，nginx也可以实现将文件上传，但是由于upload模块将文件存入时会自己命名存入，无法记录相关文件类型。如果直接从nginx存储目录下载，会导致文件的后缀丢失，无法获取文件类型。

####nginx安装脚本

[[email protected] ~]# cat autonginx 
#!/bin/bash
#install nginx/1.10.3

#Installation path
INSTALL_DIR=/usr/local/
SRC_DIR=/opt/package

[ ! -d ${INSTALL_DIR} ] && mkdir -p ${INSTALL_DIR}
[ ! -d ${SRC_DIR} ] && mkdir -p ${SRC_DIR}

# Check if user is root
if [ $(id -u) != "0" ]; then
    echo "Error: You must be root to run this script!!"
    exit 
fi

#Install dependency package
for Package in wget gcc gcc-c++ autoconf automake zlib zlib-devel openssl openssl-devel pcre pcre-devel unzip
do
    yum -y install $Package
done

function  Install_Nginx()
{
#update version
NGINX="nginx-1.10.3"
PCRE="pcre-8.40"
ZLIB="zlib-1.2.11"
OPENSSL="openssl-1.1.0e"
PURGE="ngx_cache_purge-2.3"
UPLOAD="nginx-upload-module-2.2"

NGINXFEATURES="--prefix=${INSTALL_DIR}nginx \
--user=nginx \
--group=nginx \
--with-http_ssl_module \
--with-http_gzip_static_module \
--with-http_stub_status_module \
--with-http_realip_module \
--with-stream \
--with-stream_ssl_module \
--pid-path=/var/run/nginx.pid \
--with-pcre=${SRC_DIR}/${PCRE} \
--with-zlib=${SRC_DIR}/${ZLIB} \
--with-openssl=${SRC_DIR}/${OPENSSL} \
--add-module=${SRC_DIR}/${PURGE} \
--add-module=${SRC_DIR}/${UPLOAD}
"

cd ${SRC_DIR}
#download package
echo 'Downloading NGINX'
if [ ! -f ${NGINX}.tar.gz ]
then
  wget -c http://nginx.org/download/${NGINX}.tar.gz
else
  echo 'Skipping: NGINX already downloaded'
fi

echo 'Downloading PCRE'
if [ ! -f ${PCRE}.tar.gz ]
then
  wget -c https://ftp.pcre.org/pub/pcre/${PCRE}.tar.gz     
else
  echo 'Skipping: PCRE already downloaded'
fi

echo 'Downloading ZLIB'
if [ ! -f ${ZLIB}.tar.gz ]
then
  wget -c http://zlib.net/${ZLIB}.tar.gz
else
  echo 'Skipping: ZLIB already downloaded'
fi

echo 'Downloading OPENSSL'
if [ ! -f ${OPENSSL}.tar.gz ]
then
  wget -c http://www.openssl.org/source/${OPENSSL}.tar.gz
else
  echo 'Skipping: OPENSSL already downloaded'
fi

echo 'Downloading PURGE'

if [ ! -f .tar.gz ]
then
   wget -c https://github.com//FRiCKLE/ngx_cache_purge/archive/.tar.gz
else
   echo 'Skipping: PURGE already downloaded'
fi



if [ ! -f  ]
then
   wget -c  https://codeload.github.com/vkholodkov/nginx-upload-module/zip/
else
   echo 'Skipping: nginx-upload-module already downloaded'
fi



echo '----------Unpacking downloaded archives. This process may take serveral minutes---------'

echo "Extracting ${NGINX}..."
tar xzf ${NGINX}.tar.gz
echo 'Done.'

echo "Extracting ${PCRE}..."
tar xzf ${PCRE}.tar.gz
echo 'Done.'

echo "Extracting ${ZLIB}..."
tar xzf ${ZLIB}.tar.gz
echo 'Done.'

echo "Extracting ${OPENSSL}..."
tar xzf ${OPENSSL}.tar.gz
echo 'Done.'

echo "Extracting ${PURGE}..."
tar xzf .tar.gz
echo 'Done.'


echo "Extracting ${UPLOAD}..."
unzip 
echo 'Done.'


#add new user
groupadd -r nginx
useradd -r -g nginx nginx

#make
echo '###################'
echo 'Compile NGINX'
echo '###################'
cd ${SRC_DIR}/${NGINX}
./configure ${NGINXFEATURES}
make
make install
echo '###################'
echo 'ok'
echo '###################'
}

Install_Nginx

在此前，创建/www/web/upload/tmp目录。然后配置一下内容重启nginx。

####nginx配置文件


#####upload所需要的server


server {
    listen ;

    client_max_body_size m;
    client_body_buffer_size k;
    proxy_set_header Host $host;
    proxy_set_header X-Real-IP $remote_addr;
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_set_header REMOTE_ADD $remote_addr;


    error_page  = @405;

        location / {
            index index.html index.htm index.php;
        }

        location @405
        {
            root /www/web/upload;
        }


    location /upload {
        # 转到后台处理URL,表示Nginx接收完上传的文件后，然后交给后端处理的地址
        upload_pass @python;

        # 临时保存路径, 可以使用散列
        # 上传模块接收到的文件临时存放的路径， 1 表示方式，该方式是需要在/tmp/nginx_upload下创建以0到9为目录名称的目录，上传时候会进行一个散列处理。
        upload_store /www/web/upload/tmp ;

        # 上传文件的权限，rw表示读写 r只读
        upload_store_access user:rw group:rw all:rw;

        set $upload_field_name "file";
        # upload_resumable on;

        # 这里写入http报头，pass到后台页面后能获取这里set的报头字段
        upload_set_form_field "${upload_field_name}_name" $upload_file_name;
        upload_set_form_field "${upload_field_name}_content_type" $upload_content_type;
        upload_set_form_field "${upload_field_name}_path" $upload_tmp_path;

        # Upload模块自动生成的一些信息，如文件大小与文件md5值
        upload_aggregate_form_field "${upload_field_name}_md5" $upload_file_md5;
        upload_aggregate_form_field "${upload_field_name}_size" $upload_file_size;

        # 允许的字段，允许全部可以 "^.*$"
        upload_pass_form_field "^.*$";
        # upload_pass_form_field "^submit$|^description$";

        # 每秒字节速度控制，0表示不受控制，默认0, 128K
        upload_limit_rate ;

        # 如果pass页面是以下状态码，就删除此次上传的临时文件
        upload_cleanup    -;

        # 打开开关，意思就是把前端脚本请求的参数会传给后端的脚本语言，比如：http://192.168.1.251:9000/upload/?k=23,后台可以通过POST['k']来访问。
        upload_pass_args on;
    }

    location @python {
        proxy_pass http://localhost:;
        # return 200;  # 如果不需要后端程序处理，直接返回200即可
    }
}


#######download所需要的server#############


 server {
        listen ;
        root /tmp/nginx_upload/;
        location /nginx_upload {
                root /tmp;

        }

     }

此刻，你已经完成了nginx的配置，可以使用小程序进行一下上传测试，看看nginx是否可以上传文件。

import requests


file_dir = 'images/logo.png'
file = open(file_dir, 'rb')
files = {'attachment_file': (file_dir, file, 'image/png', {})}
r1 = requests.post('http://172.18.3.183:99/upload', files=files)
print(r1.text)
file.close()

通过python模拟post请求，将图片上传到刚才nginx开放的接口，然后在刚才创建的目录查找文件

接下来，进行后端服务器的部署，可以使用django进行部署。

1.pip install Django==1.8.9

2. django-admin.py startproject UPload

3. cd UPload/

4. python manage.py startapp file_parse

5. cd file_parse

6. 将下面的view.py添加到view.py中

7. 在settings.py中添加file_parse

8. 在url.py中添加 url(r’^upload’,file_parse_views.upload),

9. python manage.py runserver 0.0.0.0:9999

#django中的view.py

import os
import json
import time


from django.views.decorators.csrf import csrf_exempt

date = time.strftime('%Y%m%d')
UPLOAD_FILE_PATH = '/tmp/nginx_upload/%s/' %date
isExists=os.path.exists(UPLOAD_FILE_PATH)
if not isExists:
         os.makedirs(UPLOAD_FILE_PATH)
else:
        print('path isexist!')


from django.shortcuts import render
from django.http import HttpResponse

# Create your views here.


def index(request):
    return HttpResponse("Hello Django!")

@csrf_exempt
def upload(request):
    request_params = request.POST
    print(request_params)
    file_name = request_params['file_name']
    file_content_type = request_params['file_content_type']
    file_md5 = request_params['file_md5']
    file_path = request_params['file_path']
    file_size = request_params['file_size']

    ip_address = request.META.get('HTTP_X_REAL_IP') or request.META.get('HTTP_REMOTE_ADD')

    # save file to tmp
    new_file_name = '%s_%s' % (file_md5,file_name)
    new_file_path = ''.join([UPLOAD_FILE_PATH, new_file_name])
    with open(new_file_path, 'a') as new_file:
        with open(file_path, 'rb') as f:
            new_file.write(f.read())

    content = json.dumps({
        'name': file_name,
        'content_type': file_content_type,
        'md5': file_md5,
        'path': file_path,
        'size': file_size,
        'ip': ip_address,
    })

    print(new_file_path)
    path_dir = new_file_path.split('/')[:]
    res = '/'.join(path_dir)
    download_url = '/'.join(('http://172.18.3.183:88',res))

    response = HttpResponse(download_url, content_type='application/json; charset=utf-8')


    return response

实现效果图：

nginx upload模块+python 后端处理模仿fastdfs实现文件存取
fastdfs Linux nginx Python django url
上一篇: FastDfs (二) Nginx 整合 Fastdfs
下一篇: 开发者问第四期问答分享来啦！

继续阅读