Graph().as_default()X=placeholder()# Create a weight matrix for 2 output classes:# One with a weight vector (1, 1) for blue and # one with a weight vector (-1, -1) for redW=Variable([[1,-1],[1,-1]])b=Variable([0,0])p=softmax(add(matmul(X,W),b))session=Session()output_probabilities=session.run(p,{X:np.concatenate((blue_points,red_points))})
classlog(Operation):"""Computes the natural logarithm of x element-wise.
"""def__init__(self,x):super().__init__([x])defcompute(self,x_value):returnnp.log(x_value)
# Create a new graphGraph().as_default()X=placeholder()c=placeholder()W=Variable([[1,-1],[1,-1]])b=Variable([0,0])p=softmax(add(matmul(X,W),b))# Cross-entropy lossJ=negative(reduce_sum(reduce_sum(multiply(c,log(p)),axis=1)))session=Session()print(session.run(J,{X:np.concatenate((blue_points,red_points)),c:[[1,0]]*len(blue_points)+[[0,1]]*len(red_points)}))
classGradientDescentOptimizer:def__init__(self,learning_rate):self.learning_rate=learning_ratedefminimize(self,loss):learning_rate=self.learning_rateclassMinimizationOperation(Operation):defcompute(self):grad_table=compute_gradients(loss)# Iterate all variablesfornodeingrad_table:iftype(node)==Variable:grad=grad_table[node]# Take a step along the direction of the negative gradientnode.value-=learning_rate*gradreturnMinimizationOperation()
fromqueueimportQueuedefcompute_gradients(loss):grad_table={}grad_table[loss]=1# Perform a breadth-first search, backwards from the lossvisited=set()queue=Queue()visited.add(loss)queue.put(loss)whilenotqueue.empty():node=queue.get()ifnode!=loss:grad_table[node]=0# Iterate all consumersforconsumerinnode.consumers:lossgrad_wrt_consumer_output=grad_table[consumer]# Retrieve the function which computes gradients with respect to# consumer's inputs given gradients with respect to consumer's output.consumer_op_type=consumer.__class__bprop=_gradient_registry[consumer_op_type]# Get the gradient of the loss with respect to all of consumer's inputslossgrads_wrt_consumer_inputs=bprop(consumer,lossgrad_wrt_consumer_output)iflen(consumer.input_nodes)==1:# If there is a single input node to the consumer, lossgrads_wrt_consumer_inputs is a scalargrad_table[node]+=lossgrads_wrt_consumer_inputselse:# Otherwise, lossgrads_wrt_consumer_inputs is an array of gradients for each input nodenode_index_in_consumer_inputs=consumer.input_nodes.index(node)# Get the gradient of the loss with respect to nodelossgrad_wrt_node=lossgrads_wrt_consumer_inputs[node_index_in_consumer_inputs]grad_table[node]+=lossgrad_wrt_nodeifhasattr(node,"input_nodes"):forinput_nodeinnode.input_nodes:ifnotinput_nodeinvisited:visited.add(input_node)queue.put(input_node)returngrad_table
importmatplotlib.pyplotaspltimportnumpyasnpimporttensorslowasts# Create two clusters of red points centered at (0, 0) and (1, 1), respectively.red_points=np.concatenate((0.2*np.random.randn(25,2)+np.array([[0,0]]*25),0.2*np.random.randn(25,2)+np.array([[1,1]]*25)))# Create two clusters of blue points centered at (0, 1) and (1, 0), respectively.blue_points=np.concatenate((0.2*np.random.randn(25,2)+np.array([[0,1]]*25),0.2*np.random.randn(25,2)+np.array([[1,0]]*25)))# Plot themplt.scatter(red_points[:,0],red_points[:,1],color='red')plt.scatter(blue_points[:,0],blue_points[:,1],color='blue')plt.show()
# Create a new graphts.Graph().as_default()# Create training input placeholderX=ts.placeholder()# Create placeholder for the training classesc=ts.placeholder()# Build a hidden layerW_hidden1=ts.Variable(np.random.randn(2,4))b_hidden1=ts.Variable(np.random.randn(4))p_hidden1=ts.sigmoid(ts.add(ts.matmul(X,W_hidden1),b_hidden1))# Build a hidden layerW_hidden2=ts.Variable(np.random.randn(4,8))b_hidden2=ts.Variable(np.random.randn(8))p_hidden2=ts.sigmoid(ts.add(ts.matmul(p_hidden1,W_hidden2),b_hidden2))# Build a hidden layerW_hidden3=ts.Variable(np.random.randn(8,2))b_hidden3=ts.Variable(np.random.randn(2))p_hidden3=ts.sigmoid(ts.add(ts.matmul(p_hidden2,W_hidden3),b_hidden3))# Build the output layerW_output=ts.Variable(np.random.randn(2,2))b_output=ts.Variable(np.random.randn(2))p_output=ts.softmax(ts.add(ts.matmul(p_hidden3,W_output),b_output))# Build cross-entropy lossJ=ts.negative(ts.reduce_sum(ts.reduce_sum(ts.multiply(c,ts.log(p_output)),axis=1)))# Build minimization opminimization_op=ts.train.GradientDescentOptimizer(learning_rate=0.03).minimize(J)# Build placeholder inputsfeed_dict={X:np.concatenate((blue_points,red_points)),c:[[1,0]]*len(blue_points)+[[0,1]]*len(red_points)}# Create sessionsession=ts.Session()# Perform 100 gradient descent stepsforstepinrange(2000):J_value=session.run(J,feed_dict)ifstep%100==0:print("Step:",step," Loss:",J_value)session.run(minimization_op,feed_dict)