Neural network overview

If you still don’t know what the artificial neural network is - you should read some information about it (e.g. http://en.wikipedia.org/wiki/Artificial_neuron).

In this article I will use artificial neurons based on radial basis functions (RBF). The influence of a neuron located in point c upon x is calculated as: $\varphi(x) = e^{\frac{||x-c||^2}{a^2}}$ where a is also a parameter of current neuron. Let us consider that the network consists on N neurons. Then we are able to calculate total influence: $u(x) = \sum\limits_{i=1}^N w_i \varphi_i(x)=\sum\limits_{i=1}^N w_i e^{\frac{||x-c_i||^2}{a_i^2}}$

The first partial differential equation that I’ll solve is: $\frac{\partial^2 U}{\partial x^2} + \frac{\partial^2 U}{\partial y^2} = 1$ $U(0) = 0$ $U(1) = 0$

At first iteration I define sets of neuron centers (funCenters), widths (funWidths), and collocation points. A collocation point is a point, where total influence of all neurons will be calculated, compared to a given value, and after that’s done, parameter corrections will be made. There are two types of collocation points: points located inside the area and points lying on it’s border.

(* Randomly selected values *)
let funCenters = [-0.3; 0.1; 0.15; 0.2; 0.3; 0.45; 0.5; 0.52; 0.6; 0.69; 0.8; 0.91; 1.03]
(* At first I suggest what width is 0.3 *)
let funWidths = List.map (fun x -&gt; 0.3) [1..13]
(* Inner collocation points *)
let collocationInside = [0.01; 0.2; 0.25; 0.41; 0.56; 0.61; 0.78; 0.8; 0.95; 0.99]
(* Border collocation points *)
let collocationOutside = [0.0; 1.0]

I declare some helpful functions:

(* calculate a^b *)
let rec pow (a:float) (b:int) =
    match b with
    | 0 -&gt; 1.0
    | _ -&gt; a*( pow a (b-1) )
 
(* generate a RBF with specified center and width *)
let funcGen (center:float) (width:float) =
    fun pos -&gt; exp (-(pow (pos-center) 2)/(pow width 2))
 
(* generate a second derivate (d^2 U / d x^2) for RBF with specified center and width *)
let ddfuncGen (center:float) (width:float)=
    fun pos -&gt; ( 4.0 * (pow (pos - center) 2) / (pow width 4) - 2.0/(pow width 2)) * exp (-(pow (pos-center) 2) / (pow width 2))
 
(* right part of PDE *)
let neodn (x:float) =
    1.0
 
(* border values of PDE *)
let funcBorder (x:float) =
    match x with
    | 0.0 -&gt; 0.0
    | 1.0 -&gt; 0.0
    | _ -&gt; 0.0

Now I need to fill a list of weights. To do that I use LSM and the dnAnalytics library.

(* first approximation of weights using LSM *)
let getFirstWeights =
    (* list of RBF functions, generated for each center-width pair by funcGen function *)
    let phiList = List.map2 funcGen funCenters funWidths
    (* list of second derivatives for RBF for each center-width pair *)
    let ddphiList = List.map2 ddfuncGen funCenters funWidths
    (* LSM *)
    (* every row i of H represents i-th collocation point, and every value j in i-th row is a contribution of j-th neuron. Z[i] = right part of PDE for inner collocation points i or border value for border collocation point *)
    let mutable H = dnAnalytics.LinearAlgebra.DenseMatrix((collocationInside |&gt; List.length) + (collocationOutside |&gt; List.length), funCenters |&gt;List.length)
    let mutable Z = DenseVector((collocationInside |&gt; List.length) + (collocationOutside |&gt; List.length))
    for i=0 to (collocationInside |&gt; List.length)-1 do
        for j=0 to (funCenters |&gt;List.length)-1 do
            H.Item(i,j) &lt;- ddphiList.[j] collocationInside.[i]
        done
        Z.Item(i)  List.length)-1 do
        for j=0 to (funCenters |&gt;List.length)-1 do
            H.Item(i+(collocationInside |&gt; List.length),j) &lt;- phiList.[j] collocationOutside.[i]
        done
        Z.Item(i) &lt;- funcBorder collocationOutside.[i]
    done
    let W = ((H.Transpose())*H).Inverse()
    let w = W*(H.Transpose())*Z
    (* you can find vectorToList function into source file *)
    vectorToList w 0

After a priori definition of weights I begin network training. I do this by fixing all parameters except one and minimizing residual value I (N — number of inner collocation points, M — number of border collocation points, K — number of neurons): $I = \sum\limits_{i=1}^{N} \left(\left(\sum\limits_{k=1}^{K} w_k\frac{\partial^2 \varphi_k(x_i)}{\partial x^2}\right) - f(x_i)\right)^2 + \lambda\sum\limits_{j=1}^{M}\left(\left(\sum\limits_{k=1}^{K} w_k \varphi_k(x_j)\right)-b(x_j)\right)^2$

Minimisation of I is made by gradient descent method so we need to calculate all partial derivatives $\frac{\partial I}{\partial a_i}\qquad \frac{\partial I}{\partial c_i}\qquad \frac{\partial I}{\partial w_i}$

(* function sum3 gets 3 lists and performs an operation f over heads of all lists, adding result into acc *)
let rec sum3 (f : float -&gt; float -&gt; float -&gt; float -&gt; float) (a : float list) (b : float list) (c : float list) x acc =
    match a with
    | [] -&gt; acc
    | _ -&gt; sum3 f (List.tl a) (List.tl b) (List.tl c) x (acc+(f (List.hd a) (List.hd b) (List.hd c) x))
 
(* training network *)
let solve =
    (* get weights for first iteration by LSM *)
    let weights = getFirstWeights
    (* loop *)
    let rec fo a b (w : float list) (az : float list) (fc : float list) =
        match a with
        | _ when a&lt; b -&gt;
                        (* residual for some inner point x (l1 -- weights, l2 -- widths, l3 -- centers list *)
                        let residualDerivative l1 l2 l3 x =
                            let rez = (sum3 (fun w a c x -&gt; w * (4.0*(x-c)*(x-c)-2.0*a*a)/(a*a*a*a)*(exp (-(x-c)*(x-c)/(a*a)))) l1 l2 l3 x 0.0) - (neodn x)
                            rez
                        (* residual for some border point x *)
                        let residual l1 l2 l3 x =
                            let rez = (sum3 (fun w a c x -&gt; w * (exp (-(x-c)*(x-c)/(a*a)))) l1 l2 l3 x 0.0) - (funcBorder x)
                            rez
                        (* dI/dw for neuron with width a and center c *)
                        let dIw (a : float) (c : float) =
                            let lambda = 100.0
                            let rez = (List.sum_by (fun x -&gt; 2.0*(residualDerivative w az fc x)*(4.0*(x-c)*(x-c)-2.0*a*a)/(a*a*a*a)*(exp (-(x-c)*(x-c)/(a*a)))) collocationInside ) + (List.sum_by (fun x -&gt; 2.0*lambda*(residual w az fc x)*(exp (-(x-c)*(x-c)/(a*a)))) collocationBorder)
                            rez
 
                        (* generate list of derivatives for current widths az and centers fc *)
                        let dIdw = List.map2 dIw az fc
                        (* update widths by gradient descent method. Step is fixed (I am planning to add adaptive step) *)
                        let weights2 = List.map2 (fun a b -&gt; a-(1e-12)*b) w dIdw
 
                        (* dI/da  for neuron with width a, center c and weight w *)
                        let dIa (w : float) (a : float) (c : float) =
                            let lambda = 100.0
                            let rez = (List.sum_by (fun x -&gt; 2.0*(residualDerivative weights2 az fc x)*w*4.0*(exp (-(x-c)*(x-c)/(a*a))*(a*a*a*a-5.0*(x-c)*(x-c)*a*a+2.0*(pow (x-c) 4))/(pow a 7))) collocationInside) + (List.sum_by (fun x -&gt; 2.0*lambda*(residual weights2 az fc x)*w*2.0*(x-c)*(x-c)/(a*a*a)*(exp (-(x-c)*(x-c)/(a*a)))) collocationBorder)
                            rez
 
                        (* generate list of derivatives for current widths2, centers fc, widths az *)
                        let dIda = List.map3 dIa weights2 az fc
 
                        (* update widths *)
                        let az2 = List.map2 (fun a b -&gt; a-(1e-12)*b) az dIda
 
                        (* dI/dc  for neuron with width a, center c and weight w *)
                        let dIс (w : float) (a : float) (c : float) =
                            let lambda = 100.0
                            let rez = (List.sum_by (fun x -&gt; 2.0*(residualDerivative weights2 az2 fc x)*w*(x-c)*(-12.0*a*a+8.0*(x-c)*(x-c))/(pow a 6)*(exp (-(x-c)*(x-c)/(a*a)))) collocationInside) + (List.sum_by (fun x -&gt; 2.0*lambda*(residual weights2 az2 fc x)*w*2.0*(x-c)/(a*a)*(exp (-(x-c)*(x-c)/(a*a)))) collocationBorder)
                            rez
 
                        (* generate list of derivatives for current widths2, centers fc, widths az2 *)
                        let dIdc = List.map3 dIс weights2 az2 fc
 
                        (* update centers *)
                        let fc2 = List.map2 (fun a b -&gt; a-(1e-12)*b) fc dIdc
 
                        fo (a+1) b weights2 az2 fc2
        | _ -&gt; ()
    fo 0 1000 weights funWidths funCenters
    (* now I make 1000 of iterations what is enought. But in the future I will change this condition *)
    (* return true *)
    true

This model is made for learning purposes and I think that now it’s very simple to analyze parameters and build a more complicated solution.

As always, source code is available.

November 10th, 2008 in Using neural networks for solving partial differential equations

fs4learningJust another Robotics.icstweb.org weblog

Neural network overview

Leave a comment

Blogroll