#查詢使用者餘額代碼案例
import sys
import MySQLdb
import pandas as pd
optmap = {
'dbuser' : 'aduser',
'dbpass' : '123654',
'dbhost' : '192.168.10.14',
'dbport' : 3306,
'dbname' : 'HBAODB'
}
def sql_select(reqsql):
try:
db_conn = MySQLdb.connect(user=optmap['dbuser'], passwd=optmap['dbpass'], host=optmap['dbhost'], port=optmap['dbport'], db=optmap['dbname'])
db_cursor=db_conn.cursor()
db_conn.query("use %s"%optmap['dbname'])
count = db_cursor.execute(reqsql)
ret = db_cursor.fetchall()
db_cursor.close()
db_conn.close
return ret
except MySQLdb.Error,e:
print "Mysql ERROR %d:%s" %(e.args[0], e.args[1])
return ''
def getusercoin(userid):
i = int(userid) % 10
reqsql = "select ID,COINREFER from CHARCOIN%u where ID=%u" % (int(i), int(userid))
#print reqsql
ret = sql_select(reqsql) #調用前面的函數
#print ret
return ret[0]
def getall(userlist):
userdata = pd.DataFrame(columns=('userid', 'coin'))
index = 0
for userid in userlist:
coins = getusercoin(userid) #調用前面的函數
#print coins[0],coins[1]/100.0
if coins[0] is not None:
userdata.loc[index] = (str(coins[0]), coins[1]/100.0)
else:
userdata.loc[index] = (str(userid), 0)
index += 1
#print userdata.tail(10)
df = spark.createDataFrame(userdata)
#df.createOrReplaceTempView('userdata')
df.show(50)
#使用者消費查詢代碼案例
import datetime
import time
'dbhost' : '192.168.10.12',
'dbname' : 'JIESUANDB'
#使用者人民币消費
def getuserconsume(userid, startday): #定義帶參函數
strdate = startday.strftime("%y%m%d")
# 送禮物 + 守護 + 點歌 + 表情貼
reqsql = "select CONSUMERID,SUM(DUBIOPNUM) from `DUBIJIESUANTONGJI_%s` where CONSUMERID=%u AND DBIOPTYPE=2 AND (OPTYPE=1 OR OPTYPE=4 OR OPTYPE=17 OR OPTYPE=25)" % (strdate, int(userid))
print reqsql
if ret[0][0] is not None:
return float(ret[0][1])/100.0
else:
return 0
#使用者充值
def getusercharge(userid, startday):
reqsql = "select CONSUMERID,SUM(DUBIOPNUM) from `DUBIJIESUANTONGJI_%s` where CONSUMERID=%u AND DUBIOPTYPE=1 AND (OPTYPE=1016 OR OPTYPE=1020 OR OPTYPE=1021)" % (strdate, int(userid))
ret = sql_select(reqsql)#調用前面的函數
print ret
#使用者當天結餘人民币
def getusercurcoin(userid, startday):
reqsql = "select CONSUMERID,CURRENTNUM from `DUBIJIESUANTONGJI_%s` where CONSUMERID=%u ORDER BY OPTIME DESC LIMIT 1" % (strdate, int(userid))
ret = sql_select(reqsql)
if ret:
def getconsume():
startdate = datetime.date(2017, 1, 1)
enddate = datetime.date(2017, 2, 2)
userid = 3101011990
userdata = pd.DataFrame(columns=('date','userid','charge', 'consume', 'dayleftcoin'))
# 計算日差
td = enddate - startdate
datelen = td.days + 1
#print datelen
delta = datetime.timedelta(days=1)
allcoins = 0
for i in range(0,datelen):
startday = startdate + delta * i
consume_coin = getuserconsume(userid, startday)#調用前面的函數
charge = getusercharge(userid, startday)#調用前面的函數
dayleftcoin = getusercurcoin(userid, startday)#調用前面的函數
userdata.loc[index] = (startday.strftime("%Y-%m-%d"),str(userid), charge, consume_coin, dayleftcoin)
#userdata.loc[index] = ('total',str(userid), allcoins, 0)
print userdata.tail(100)
return
getconsume()
#查詢使用者機器ID 代碼案例
'dbhost' : '192.168.10.15',
'dbname' : 'JIQIDB'
def getusermid(userid, months):
i = int(userid) % 50
reqsql = "select USERID,MACHINEID from LOGINHISTORY%s%u where USERID=%u group by MACHINEID" % (months,int(i), int(userid))
#print ret[0]
return ret
today = datetime.date.today()
months = today.strftime("%Y%m")
userdata = pd.DataFrame(columns=('USERID', 'MACHINEID'))
coins = getusermid(userid, months)
for i in range(len(coins)):
#print coins[i]
userdata.loc[index] = (str(coins[i][0]), str(coins[i][1]))
index += 1
#userdata.loc[index] = (str(coins[0]), coins[1]/100.0)
#index += 1
df.show(1000)
#人民币統計代碼案例
from pyspark.sql import Row
from pyspark.sql.types import *
from pyspark.sql.functions import udf
import mysql_op
from mysql_op import MySQL
import numpy as np
from fastparquet import ParquetFile
from fastparquet import write
def fromDayToDay(startdate, datelen, func):
endday = startdate + delta * (i + 1)
func(startday, endday)
def fromDayToEndDay(startdate, datelen, endday, func):
#endday = startdate + delta * (i + 1)
# 擷取人民币資料
def saveDayPackageData(startday, endday):
#資料庫連接配接參數
dbconfig = {'host':'192.168.10.12',
'port': 3306,
'user':'user',
'passwd':'123654',
'db':'JIESUANDB',
'charset':'utf8'}
#連接配接資料庫，建立這個類的執行個體
mysql_cn= MySQLdb.connect(host=dbconfig['host'], port=dbconfig['port'],user=dbconfig['user'], passwd=dbconfig['passwd'], db=dbconfig['db'])
strday = startday.strftime("%Y%m%d")
tsstart=time.mktime(startday.timetuple())
tsend=time.mktime(endday.timetuple())
sql = "SELECT OPTIME,CONSUMERID,PRESERVENUM,CURRENTNUM,DUBIOPTYPE,DUBIOPNUM,OPTYPE,OPDETAIL,OPNUM FROM `DUBIJIESUANTONGJI_%s`" % (strdate)
print sql
pddf = pd.read_sql(sql, con=mysql_cn)
mysql_cn.close()
print pddf.head(5)
dflen = len(pddf.index)
if dflen > 0:
print pddf.describe()
write("/home/haoren/logstatis/billdata"+strday+".parq", pddf)
def savePackageData():
startday = datetime.date(2016, 12, 28)
endday = datetime.date(2016, 12, 28)
td = endday - startday
# 擷取包裹資料
fromDayToDay(startday, datelen, saveDayPackageData)
# 擷取WF冊資料
def saveDayWifiPhoneRegData(startday, endday):
dbconfig = {'host':'192.168.10.15',
'db':'AADB',
sql = "select USERID from NEW_WEB_USER where TIME< %d AND TYPE=17" % (tsend)
write("/home/haoren/logstatis/wifiphonereg"+strday+".parq", pddf)
def saveWifiPhoneReg():
startday = datetime.date(2016, 12, 1)
endday = datetime.date(2016, 12, 1)
fromDayToDay(startday, datelen, saveDayWifiPhoneRegData)
OPTypeName = {
0:"會員",
1:"道具",
}
OpDetailName19 = {
1:"購物儲存收益",
2:"下注和返注",
3:"發紅包",
4:"搶紅包",
OpDetailName22 = {
1:"活動1收益到總賬号",
2:"活動2收益到總賬号",
3:"活動3收益到總賬号",
OpDetailName23 = {
0:"購買會員",
1:"購買道具",
2:"掃雷",
def getOpTypeName(func):
name = OPTypeName.get(func)
if name == None:
return ""
return name.decode('utf8')
def getOpDetailName(func, detail):
if func == 19:
if detail > 10000 and detail < 30000:
return "包裹復原".decode('utf8')
elif detail > 50000 and detail < 60000:
return "紅包接龍".decode('utf8')
name = OpDetailName19.get(detail)
if name == None:
return ""
else:
return name.decode('utf8')
elif func == 22:
name = OpDetailName22.get(detail)
elif func == 23:
name = OpDetailName23.get(detail)
def getDayPackageData(startday, endday):
print strday + '人民币資料'
df = spark.read.load("/home/haoren/logstatis/billdata"+strday+".parq")
df.show(10)
#df.createOrReplaceTempView('billdata')
#df.registerTempTable("billdata")
#sqlret = sqlc.sql("SELECT count(*) from billdata")
#sqlret.show(1)
df2 = df.withColumn('OPTYPENAME', udf(getOpTypeName)(df.OPTYPE))
df2.show(10)
df = df2.withColumn('DETAILNAME', udf(getOpDetailName)(df2.OPTYPE, df2.OPDETAIL))
df.createOrReplaceTempView('billdata')
def getPackageData():
fromDayToDay(startday, datelen, getDayPackageData)#調用前面的函數
print 'getPackageData finish'
# 擷取充值資料
def getChargeInfo(startday, endday):
dbconfig = {'host':'192.168.10.14',
'port': 3306,
'user':'user',
'passwd':'123654',
'db':'BAOIMDB',
'charset':'utf8'}
regdata = pd.DataFrame()
for i in range(0, 20):
sql = "SELECT * FROM `USERCONSUMPTIONRECORD%d` where TIME > %d AND TIME < %d" % (i, tsstart, tsend)
print sql
#pddf = pd.DataFrame()
pddf = pd.read_sql(sql, con=mysql_cn)
#print pddf.head(5)
if len(pddf.index) > 0:
regdata = regdata.append(pddf,ignore_index=True)
print regdata.tail(5)
if len(regdata.index) > 0:
print regdata.describe()
write("/home/haoren/logstatis/register"+strday+".parq", regdata)
def pudf(x):
return getOpTypeName(x.OPTYPE)
def getMergeData(strday):
dfbill = ParquetFile("/home/haoren/logstatis/billdata"+strday+".parq").to_pandas()
dfwifireg = ParquetFile("/home/haoren/logstatis/wifiphonereg"+strday+".parq").to_pandas()
tempdf = pd.merge(dfbill, dfwifireg, left_on='CONSUMERID', right_on='USERID')
#write("/home/haoren/logstatis/analyze"+strday+".parq", tempdf)
#print tempdf.head(10)
tempdf['OPTYPENAME'] = tempdf.apply(lambda x:getOpTypeName(x.OPTYPE), axis=1)
tempdf['DETAILNAME'] = tempdf.apply(lambda x:getOpDetailName(x.OPTYPE,x.OPDETAIL), axis=1)
df = spark.createDataFrame(tempdf)
return df
def analyzeDayBillData(startday, endday):
dfwifireg = spark.read.load("/home/haoren/logstatis/wifiphonereg"+strday+".parq")
df3 = df.join(dfwifireg, df.CONSUMERID == dfwifireg.USERID)
df3.show(10)
df3.write.parquet("/home/haoren/logstatis/analyze"+strday+".parq")
#df2 = df3.withColumn('OPTYPENAME', udf(getOpTypeName)(df3.OPTYPE))
#df2.show(10)
#df = df2.withColumn('DETAILNAME', udf(getOpDetailName)(df2.OPTYPE, df2.OPDETAIL))
#df.show(10)
#df.createOrReplaceTempView('analyzebilldata')
def analyzeDayBillData2(startday, endday):
#df = spark.read.load("/home/haoren/logstatis/analyze"+strday+".parq")
df = getMergeData(strday)
df.createOrReplaceTempView('analyzebilldata')
def analyzeBillData():
fromDayToDay(startday, datelen, analyzeDayBillData2)#調用前面的函數
print 'analyzeBillData finish'
savePackageData()
getPackageData()
#saveWifiPhoneReg()
#analyzeBillData()