Skip to content

Commit da449e0

Browse files
RiptideBoRiptideBo
authored
Merge pull request #1 from RiptideBo/stephen_branch
stephen_branch
2 parents e1befed + 6e61ac1 commit da449e0

File tree

2 files changed

+336
-13
lines changed

2 files changed

+336
-13
lines changed

Diff for: ‎Neural_Network/convolution_neural_network.py

+305
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,305 @@
1+
#-*- coding: utf-8 -*-
2+
3+
'''
4+
- - - - - -- - - - - - - - - - - - - - - - - - - - - - -
5+
Name - - CNN - Convolution Neural Network For Photo Recognizing
6+
Goal - - Recognize Handing Writting Word Photo
7+
Detail：Total 5 layers neural network
8+
* Convolution layer
9+
* Pooling layer
10+
* Input layer layer of BP
11+
* Hiden layer of BP
12+
* Output layer of BP
13+
Author: Stephen Lee
14+
Github: [email protected]
15+
Date: 2017.9.20
16+
- - - - - -- - - - - - - - - - - - - - - - - - - - - - -
17+
'''
18+
19+
import numpy as np
20+
import matplotlib.pyplot as plt
21+
22+
class CNN():
23+
24+
def __init__(self,conv1_get,size_p1,bp_num1,bp_num2,bp_num3,rate_w=0.2,rate_t=0.2):
25+
'''
26+
:param conv1_get: [a,c,d]，size, number, step of convolution kernel
27+
:param size_p1: pooling size
28+
:param bp_num1: units number of flatten layer
29+
:param bp_num2: units number of hidden layer
30+
:param bp_num3: units number of output layer
31+
:param rate_w: rate of weight learning
32+
:param rate_t: rate of threshold learning
33+
'''
34+
self.num_bp1 = bp_num1
35+
self.num_bp2 = bp_num2
36+
self.num_bp3 = bp_num3
37+
self.conv1 = conv1_get[:2]
38+
self.step_conv1 = conv1_get[2]
39+
self.size_pooling1 = size_p1
40+
self.rate_weight = rate_w
41+
self.rate_thre = rate_t
42+
self.w_conv1 = [np.mat(-1*np.random.rand(self.conv1[0],self.conv1[0])+0.5) for i in range(self.conv1[1])]
43+
self.wkj = np.mat(-1 * np.random.rand(self.num_bp3, self.num_bp2) + 0.5)
44+
self.vji = np.mat(-1*np.random.rand(self.num_bp2, self.num_bp1)+0.5)
45+
self.thre_conv1 = -2*np.random.rand(self.conv1[1])+1
46+
self.thre_bp2 = -2*np.random.rand(self.num_bp2)+1
47+
self.thre_bp3 = -2*np.random.rand(self.num_bp3)+1
48+
49+
50+
def save_model(self,save_path):
51+
#save model dict with pickle
52+
import pickle
53+
model_dic = {'num_bp1':self.num_bp1,
54+
'num_bp2':self.num_bp2,
55+
'num_bp3':self.num_bp3,
56+
'conv1':self.conv1,
57+
'step_conv1':self.step_conv1,
58+
'size_pooling1':self.size_pooling1,
59+
'rate_weight':self.rate_weight,
60+
'rate_thre':self.rate_thre,
61+
'w_conv1':self.w_conv1,
62+
'wkj':self.wkj,
63+
'vji':self.vji,
64+
'thre_conv1':self.thre_conv1,
65+
'thre_bp2':self.thre_bp2,
66+
'thre_bp3':self.thre_bp3}
67+
with open(save_path, 'wb') as f:
68+
pickle.dump(model_dic, f)
69+
70+
print('Model saved： %s'% save_path)
71+
72+
@classmethod
73+
def ReadModel(cls,model_path):
74+
#read saved model
75+
import pickle
76+
with open(model_path, 'rb') as f:
77+
model_dic = pickle.load(f)
78+
79+
conv_get= model_dic.get('conv1')
80+
conv_get.append(model_dic.get('step_conv1'))
81+
size_p1 = model_dic.get('size_pooling1')
82+
bp1 = model_dic.get('num_bp1')
83+
bp2 = model_dic.get('num_bp2')
84+
bp3 = model_dic.get('num_bp3')
85+
r_w = model_dic.get('rate_weight')
86+
r_t = model_dic.get('rate_thre')
87+
#create model instance
88+
conv_ins = CNN(conv_get,size_p1,bp1,bp2,bp3,r_w,r_t)
89+
#modify model parameter
90+
conv_ins.w_conv1 = model_dic.get('w_conv1')
91+
conv_ins.wkj = model_dic.get('wkj')
92+
conv_ins.vji = model_dic.get('vji')
93+
conv_ins.thre_conv1 = model_dic.get('thre_conv1')
94+
conv_ins.thre_bp2 = model_dic.get('thre_bp2')
95+
conv_ins.thre_bp3 = model_dic.get('thre_bp3')
96+
return conv_ins
97+
98+
99+
def sig(self,x):
100+
return 1 / (1 + np.exp(-1*x))
101+
102+
def do_round(self,x):
103+
return round(x, 3)
104+
105+
def convolute(self,data,convs,w_convs,thre_convs,conv_step):
106+
#convolution process
107+
size_conv = convs[0]
108+
num_conv =convs[1]
109+
size_data = np.shape(data)[0]
110+
#get the data slice of original image data, data_focus
111+
data_focus = []
112+
for i_focus in range(0, size_data - size_conv + 1, conv_step):
113+
for j_focus in range(0, size_data - size_conv + 1, conv_step):
114+
focus = data[i_focus:i_focus + size_conv, j_focus:j_focus + size_conv]
115+
data_focus.append(focus)
116+
#caculate the feature map of every single kernel, and saved as list of matrix
117+
data_featuremap = []
118+
Size_FeatureMap = int((size_data - size_conv) / conv_step + 1)
119+
for i_map in range(num_conv):
120+
featuremap = []
121+
for i_focus in range(len(data_focus)):
122+
net_focus = np.sum(np.multiply(data_focus[i_focus], w_convs[i_map])) - thre_convs[i_map]
123+
featuremap.append(self.sig(net_focus))
124+
featuremap = np.asmatrix(featuremap).reshape(Size_FeatureMap, Size_FeatureMap)
125+
data_featuremap.append(featuremap)
126+
127+
#expanding the data slice to One dimenssion
128+
focus1_list = []
129+
for each_focus in data_focus:
130+
focus1_list.extend(self.Expand_Mat(each_focus))
131+
focus_list = np.asarray(focus1_list)
132+
return focus_list,data_featuremap
133+
134+
def pooling(self,featuremaps,size_pooling,type='average_pool'):
135+
#pooling process
136+
size_map = len(featuremaps[0])
137+
size_pooled = int(size_map/size_pooling)
138+
featuremap_pooled = []
139+
for i_map in range(len(featuremaps)):
140+
map = featuremaps[i_map]
141+
map_pooled = []
142+
for i_focus in range(0,size_map,size_pooling):
143+
for j_focus in range(0, size_map, size_pooling):
144+
focus = map[i_focus:i_focus + size_pooling, j_focus:j_focus + size_pooling]
145+
if type == 'average_pool':
146+
#average pooling
147+
map_pooled.append(np.average(focus))
148+
elif type == 'max_pooling':
149+
#max pooling
150+
map_pooled.append(np.max(focus))
151+
map_pooled = np.asmatrix(map_pooled).reshape(size_pooled,size_pooled)
152+
featuremap_pooled.append(map_pooled)
153+
return featuremap_pooled
154+
155+
def _expand(self,datas):
156+
#expanding three dimension data to one dimension list
157+
data_expanded = []
158+
for i in range(len(datas)):
159+
shapes = np.shape(datas[i])
160+
data_listed = datas[i].reshape(1,shapes[0]*shapes[1])
161+
data_listed = data_listed.getA().tolist()[0]
162+
data_expanded.extend(data_listed)
163+
data_expanded = np.asarray(data_expanded)
164+
return data_expanded
165+
166+
def _expand_mat(self,data_mat):
167+
#expanding matrix to one dimension list
168+
data_mat = np.asarray(data_mat)
169+
shapes = np.shape(data_mat)
170+
data_expanded = data_mat.reshape(1,shapes[0]*shapes[1])
171+
return data_expanded
172+
173+
def _calculate_gradient_from_pool(self,out_map,pd_pool,num_map,size_map,size_pooling):
174+
'''
175+
calcluate the gradient from the data slice of pool layer
176+
pd_pool: list of matrix
177+
out_map: the shape of data slice(size_map*size_map)
178+
return: pd_all: list of matrix, [num, size_map, size_map]
179+
'''
180+
pd_all = []
181+
i_pool = 0
182+
for i_map in range(num_map):
183+
pd_conv1 = np.ones((size_map, size_map))
184+
for i in range(0, size_map, size_pooling):
185+
for j in range(0, size_map, size_pooling):
186+
pd_conv1[i:i + size_pooling, j:j + size_pooling] = pd_pool[i_pool]
187+
i_pool = i_pool + 1
188+
pd_conv2 = np.multiply(pd_conv1,np.multiply(out_map[i_map],(1-out_map[i_map])))
189+
pd_all.append(pd_conv2)
190+
return pd_all
191+
192+
def trian(self,patterns,datas_train, datas_teach, n_repeat, error_accuracy,draw_e = bool):
193+
#model traning
194+
print('----------------------Start Training-------------------------')
195+
print(' - - Shape: Train_Data ',np.shape(datas_train))
196+
print(' - - Shape: Teach_Data ',np.shape(datas_teach))
197+
rp = 0
198+
all_mse = []
199+
mse = 10000
200+
while rp < n_repeat and mse >= error_accuracy:
201+
alle = 0
202+
print('-------------Learning Time %d--------------'%rp)
203+
for p in range(len(datas_train)):
204+
#print('------------Learning Image: %d--------------'%p)
205+
data_train = np.asmatrix(datas_train[p])
206+
data_teach = np.asarray(datas_teach[p])
207+
data_focus1,data_conved1 = self.convolute(data_train,self.conv1,self.w_conv1,
208+
self.thre_conv1,conv_step=self.step_conv1)
209+
data_pooled1 = self.pooling(data_conved1,self.size_pooling1)
210+
shape_featuremap1 = np.shape(data_conved1)
211+
'''
212+
print(' -----original shape ', np.shape(data_train))
213+
print(' ---- after convolution ',np.shape(data_conv1))
214+
print(' -----after pooling ',np.shape(data_pooled1))
215+
'''
216+
data_bp_input = self._expand(data_pooled1)
217+
bp_out1 = data_bp_input
218+
219+
bp_net_j = np.dot(bp_out1,self.vji.T) - self.thre_bp2
220+
bp_out2 = self.sig(bp_net_j)
221+
bp_net_k = np.dot(bp_out2 ,self.wkj.T) - self.thre_bp3
222+
bp_out3 = self.sig(bp_net_k)
223+
224+
#--------------Model Leaning ------------------------
225+
# calcluate error and gradient---------------
226+
pd_k_all = np.multiply((data_teach - bp_out3), np.multiply(bp_out3, (1 - bp_out3)))
227+
pd_j_all = np.multiply(np.dot(pd_k_all,self.wkj), np.multiply(bp_out2, (1 - bp_out2)))
228+
pd_i_all = np.dot(pd_j_all,self.vji)
229+
230+
pd_conv1_pooled = pd_i_all / (self.size_pooling1*self.size_pooling1)
231+
pd_conv1_pooled = pd_conv1_pooled.T.getA().tolist()
232+
pd_conv1_all = self._calculate_gradient_from_pool(data_conved1,pd_conv1_pooled,shape_featuremap1[0],
233+
shape_featuremap1[1],self.size_pooling1)
234+
#weight and threshold learning process---------
235+
#convolution layer
236+
for k_conv in range(self.conv1[1]):
237+
pd_conv_list = self._expand_mat(pd_conv1_all[k_conv])
238+
delta_w = self.rate_weight * np.dot(pd_conv_list,data_focus1)
239+
240+
self.w_conv1[k_conv] = self.w_conv1[k_conv] + delta_w.reshape((self.conv1[0],self.conv1[0]))
241+
242+
self.thre_conv1[k_conv] = self.thre_conv1[k_conv] - np.sum(pd_conv1_all[k_conv]) * self.rate_thre
243+
#all connected layer
244+
self.wkj = self.wkj + pd_k_all.T * bp_out2 * self.rate_weight
245+
self.vji = self.vji + pd_j_all.T * bp_out1 * self.rate_weight
246+
self.thre_bp3 = self.thre_bp3 - pd_k_all * self.rate_thre
247+
self.thre_bp2 = self.thre_bp2 - pd_j_all * self.rate_thre
248+
# calculate the sum error of all single image
249+
errors = np.sum(abs((data_teach - bp_out3)))
250+
alle = alle + errors
251+
#print(' ----Teach ',data_teach)
252+
#print(' ----BP_output ',bp_out3)
253+
rp = rp + 1
254+
mse = alle/patterns
255+
all_mse.append(mse)
256+
def draw_error():
257+
yplot = [error_accuracy for i in range(int(n_repeat * 1.2))]
258+
plt.plot(all_mse, '+-')
259+
plt.plot(yplot, 'r--')
260+
plt.xlabel('Learning Times')
261+
plt.ylabel('All_mse')
262+
plt.grid(True, alpha=0.5)
263+
plt.show()
264+
print('------------------Training Complished---------------------')
265+
print(' - - Training epoch: ', rp, ' - - Mse: %.6f' % mse)
266+
if draw_e:
267+
draw_error()
268+
return mse
269+
270+
def predict(self,datas_test):
271+
#model predict
272+
produce_out = []
273+
print('-------------------Start Testing-------------------------')
274+
print(' - - Shape: Test_Data ',np.shape(datas_test))
275+
for p in range(len(datas_test)):
276+
data_test = np.asmatrix(datas_test[p])
277+
data_focus1, data_conved1 = self.convolute(data_test, self.conv1, self.w_conv1,
278+
self.thre_conv1, conv_step=self.step_conv1)
279+
data_pooled1 = self.pooling(data_conved1, self.size_pooling1)
280+
data_bp_input = self._expand(data_pooled1)
281+
282+
bp_out1 = data_bp_input
283+
bp_net_j = bp_out1 * self.vji.T - self.thre_bp2
284+
bp_out2 = self.sig(bp_net_j)
285+
bp_net_k = bp_out2 * self.wkj.T - self.thre_bp3
286+
bp_out3 = self.sig(bp_net_k)
287+
produce_out.extend(bp_out3.getA().tolist())
288+
res = [list(map(self.do_round,each)) for each in produce_out]
289+
return np.asarray(res)
290+
291+
def convolution(self,data):
292+
#return the data of image after convoluting process so we can check it out
293+
data_test = np.asmatrix(data)
294+
data_focus1, data_conved1 = self.convolute(data_test, self.conv1, self.w_conv1,
295+
self.thre_conv1, conv_step=self.step_conv1)
296+
data_pooled1 = self.pooling(data_conved1, self.size_pooling1)
297+
298+
return data_conved1,data_pooled1
299+
300+
301+
if __name__ == '__main__':
302+
pass
303+
'''
304+
I will put the example on other file
305+
'''

Diff for: ‎Neural_Network/neuralnetwork_bp3.py

+31-13
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
import numpy as np
1010
import matplotlib.pyplot as plt
1111

12-
class Bpnw():
12+
class Bpnn():
1313

1414
def __init__(self,n_layer1,n_layer2,n_layer3,rate_w=0.3,rate_t=0.3):
1515
'''
@@ -38,7 +38,7 @@ def sig_plain(self,x):
3838
def do_round(self,x):
3939
return round(x, 3)
4040

41-
def trian(self,patterns,data_train, data_teach, n_repeat, error_accuracy,draw_e = bool):
41+
def trian(self,patterns,data_train, data_teach, n_repeat, error_accuracy, draw_e=False):
4242
'''
4343
:param patterns: the number of patterns
4444
:param data_train: training data x; numpy.ndarray
@@ -49,9 +49,9 @@ def trian(self,patterns,data_train, data_teach, n_repeat, error_accuracy,draw_e
4949
'''
5050
data_train = np.asarray(data_train)
5151
data_teach = np.asarray(data_teach)
52-
print('-------------------Start Training-------------------------')
53-
print(' - - Shape: Train_Data ',np.shape(data_train))
54-
print(' - - Shape: Teach_Data ',np.shape(data_teach))
52+
# print('-------------------Start Training-------------------------')
53+
# print(' - - Shape: Train_Data ',np.shape(data_train))
54+
# print(' - - Shape: Teach_Data ',np.shape(data_teach))
5555
rp = 0
5656
all_mse = []
5757
mse = 10000
@@ -95,9 +95,9 @@ def draw_error():
9595
plt.ylabel('All_mse')
9696
plt.grid(True,alpha = 0.7)
9797
plt.show()
98-
print('------------------Training Complished---------------------')
99-
print(' - - Training epoch: ', rp, ' - - Mse: %.6f'%mse)
100-
print(' - - Last Output: ', final_out3)
98+
# print('------------------Training Complished---------------------')
99+
# print(' - - Training epoch: ', rp, ' - - Mse: %.6f'%mse)
100+
# print(' - - Last Output: ', final_out3)
101101
if draw_e:
102102
draw_error()
103103

@@ -108,9 +108,9 @@ def predict(self,data_test):
108108
'''
109109
data_test = np.asarray(data_test)
110110
produce_out = []
111-
print('-------------------Start Testing-------------------------')
112-
print(' - - Shape: Test_Data ',np.shape(data_test))
113-
print(np.shape(data_test))
111+
# print('-------------------Start Testing-------------------------')
112+
# print(' - - Shape: Test_Data ',np.shape(data_test))
113+
# print(np.shape(data_test))
114114
for g in range(np.shape(data_test)[0]):
115115

116116
net_i = data_test[g]
@@ -127,8 +127,26 @@ def predict(self,data_test):
127127

128128

129129
def main():
130-
#I will fish the mian function later
131-
pass
130+
#example data
131+
data_x = [[1,2,3,4],
132+
[5,6,7,8],
133+
[2,2,3,4],
134+
[7,7,8,8]]
135+
data_y = [[1,0,0,0],
136+
[0,1,0,0],
137+
[0,0,1,0],
138+
[0,0,0,1]]
139+
140+
test_x = [[1,2,3,4],
141+
[3,2,3,4]]
142+
143+
#building network model
144+
model = Bpnn(4,10,4)
145+
#training the model
146+
model.trian(patterns=4,data_train=data_x,data_teach=data_y,
147+
n_repeat=100,error_accuracy=0.1,draw_e=True)
148+
#predicting data
149+
model.predict(test_x)
132150

133151
if __name__ == '__main__':
134152
main()

0 commit comments

Comments
 (0)