{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "## Project: Deep Learning\n", "#### by Yupeng Wu, Zhifang Zheng, Ze Gong\n", "#### Created using Julia 1.1.1" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m\u001b[1m Updating\u001b[22m\u001b[39m registry at `C:\\Users\\simon\\.julia\\registries\\General`" ] }, { "name": "stderr", "output_type": "stream", "text": [ "┌ Warning: Pkg.clone is only kept for legacy CI script reasons, please use `add`\n", "└ @ Pkg.API C:\\cygwin\\home\\Administrator\\buildbot\\worker\\package_win64\\build\\usr\\share\\julia\\stdlib\\v1.1\\Pkg\\src\\API.jl:391\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "\u001b[32m\u001b[1m Updating\u001b[22m\u001b[39m git-repo `https://github.com/JuliaRegistries/General.git`\n", "\u001b[2K\u001b[36m\u001b[1mFetching:\u001b[22m\u001b[39m [========================================>] 100.0 %.0 %\u001b[?25h\u001b[32m\u001b[1m Updating\u001b[22m\u001b[39m git-repo `https://github.com/invenia/Keras.jl`\n", "\u001b[?25l\u001b[2K\u001b[?25h" ] }, { "name": "stderr", "output_type": "stream", "text": [ "┌ Info: Assigning UUID 031d7243-eb6a-5793-a290-e3c72503329c to Keras\n", "└ @ Pkg.Types C:\\cygwin\\home\\Administrator\\buildbot\\worker\\package_win64\\build\\usr\\share\\julia\\stdlib\\v1.1\\Pkg\\src\\Types.jl:841\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m\u001b[1m Resolving\u001b[22m\u001b[39m package versions..." ] }, { "name": "stderr", "output_type": "stream", "text": [ "┌ Info: Path `C:\\Users\\simon\\.julia\\dev\\Keras` exists and looks like the correct package, using existing path\n", "└ @ Pkg.Types C:\\cygwin\\home\\Administrator\\buildbot\\worker\\package_win64\\build\\usr\\share\\julia\\stdlib\\v1.1\\Pkg\\src\\Types.jl:645\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", "\u001b[32m\u001b[1m Updating\u001b[22m\u001b[39m `C:\\Users\\simon\\.julia\\environments\\v1.1\\Project.toml`\n", "\u001b[90m [no changes]\u001b[39m\n", "\u001b[32m\u001b[1m Updating\u001b[22m\u001b[39m `C:\\Users\\simon\\.julia\\environments\\v1.1\\Manifest.toml`\n", "\u001b[90m [no changes]\u001b[39m\n", "\u001b[32m\u001b[1m Building\u001b[22m\u001b[39m Keras → `C:\\Users\\simon\\.julia\\dev\\Keras\\deps\\build.log`\n", "\u001b[32m\u001b[1m Building\u001b[22m\u001b[39m Conda ─→ `C:\\Users\\simon\\.julia\\packages\\Conda\\kLXeC\\deps\\build.log`\n", "\u001b[32m\u001b[1m Building\u001b[22m\u001b[39m PyCall → `C:\\Users\\simon\\.julia\\packages\\PyCall\\ttONZ\\deps\\build.log`\n", "\u001b[32m\u001b[1m Building\u001b[22m\u001b[39m Conda ─→ `C:\\Users\\simon\\.julia\\packages\\Conda\\kLXeC\\deps\\build.log`\n", "\u001b[32m\u001b[1m Building\u001b[22m\u001b[39m PyCall → `C:\\Users\\simon\\.julia\\packages\\PyCall\\ttONZ\\deps\\build.log`\n", "\u001b[32m\u001b[1m Building\u001b[22m\u001b[39m Keras ─→ `C:\\Users\\simon\\.julia\\dev\\Keras\\deps\\build.log`\n", "\u001b[32m\u001b[1m Resolving\u001b[22m\u001b[39m package versions...\n" ] } ], "source": [ "using Pkg\n", "Pkg.clone(\"https://github.com/invenia/Keras.jl\")\n", "\n", "ENV[\"PYTHON\"] = \"\"\n", "\n", "Pkg.build(\"PyCall\")\n", "Pkg.build(\"Keras\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "After successfully set the environment, use Pkg.add to add \"Keras\" and \"StatsBase\" package to use." ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m\u001b[1m Resolving\u001b[22m\u001b[39m package versions...\n", "\u001b[32m\u001b[1m Updating\u001b[22m\u001b[39m `C:\\Users\\simon\\.julia\\environments\\v1.1\\Project.toml`\n", "\u001b[90m [no changes]\u001b[39m\n", "\u001b[32m\u001b[1m Updating\u001b[22m\u001b[39m `C:\\Users\\simon\\.julia\\environments\\v1.1\\Manifest.toml`\n", "\u001b[90m [no changes]\u001b[39m\n", "\u001b[32m\u001b[1m Resolving\u001b[22m\u001b[39m" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Using Theano backend.\r\n", "WARNING (theano.configdefaults): g++ not available, if using conda: `conda install m2w64-toolchain`\r\n", "C:\\Users\\simon\\.julia\\conda\\3\\lib\\site-packages\\theano\\configdefaults.py:560: UserWarning: DeprecationWarning: there is no c++ compiler.This is deprecated and with Theano 0.11 a c++ compiler will be mandatory\r\n", " warnings.warn(\"DeprecationWarning: there is no c++ compiler.\"\r\n", "WARNING (theano.configdefaults): g++ not detected ! Theano will be unable to execute optimized C-implementations (for both CPU and GPU) and will default to Python implementations. Performance will be severely degraded. To remove this warning, set Theano flags cxx to an empty string.\r\n", "WARNING (theano.tensor.blas): Using NumPy C-API based implementation for BLAS functions.\r\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " package versions...\n", "\u001b[32m\u001b[1m Updating\u001b[22m\u001b[39m `C:\\Users\\simon\\.julia\\environments\\v1.1\\Project.toml`\n", "\u001b[90m [no changes]\u001b[39m\n", "\u001b[32m\u001b[1m Updating\u001b[22m\u001b[39m `C:\\Users\\simon\\.julia\\environments\\v1.1\\Manifest.toml`\n", "\u001b[90m [no changes]\u001b[39m\n" ] } ], "source": [ "Pkg.add(\"Keras\")\n", "using Keras\n", "Pkg.add(\"StatsBase\")\n", "using StatsBase" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "import Keras.Layers: Dense, Activation" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Now we are able to set the Neural Network model by the Sequential model API.\n", "\n", "The Sequential model is a linear stack of layers.\n", "\n", "You can create a Sequential model by passing a list of layer instances to the constructor and also simply add layers via the add method." ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "┌ Warning: `getindex(o::PyObject, s::Symbol)` is deprecated in favor of dot overloading (`getproperty`) so elements should now be accessed as e.g. `o.s` instead of `o[:s]`.\n", "│ caller = Sequential() at models.jl:39\n", "└ @ Keras C:\\Users\\simon\\.julia\\dev\\Keras\\src\\models.jl:39\n" ] }, { "data": { "text/plain": [ "Sequential(PyObject , Layer[])" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "model = Sequential()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "The model needs to know what input shape it should expect. \n", "\n", "For this reason, the first layer in a Sequential model needs to receive information about its input shape. \n", "\n", "(Only the first layer needs to do this, because following layers can do automatic shape inference)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "In this case, we are creating a 2D layers by Dense, which support the specification of their input shape via the argument input_dim. Here we set the input_dim with 700 and the input layer with 80 nodes." ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "┌ Warning: `getindex(o::PyObject, s::Symbol)` is deprecated in favor of dot overloading (`getproperty`) so elements should now be accessed as e.g. `o.s` instead of `o[:s]`.\n", "│ caller = (::getfield(Keras.Layers, Symbol(\"##Dense#40#42\")))(::Base.Iterators.Pairs{Symbol,Int64,Tuple{Symbol},NamedTuple{(:input_dim,),Tuple{Int64}}}, ::Type, ::Int64) at layers.jl:168\n", "└ @ Keras.Layers C:\\Users\\simon\\.julia\\dev\\Keras\\src\\layers.jl:168\n", "┌ Warning: `getindex(o::PyObject, s::Symbol)` is deprecated in favor of dot overloading (`getproperty`) so elements should now be accessed as e.g. `o.s` instead of `o[:s]`.\n", "│ caller = add!(::Sequential, ::Dense) at models.jl:60\n", "└ @ Keras C:\\Users\\simon\\.julia\\dev\\Keras\\src\\models.jl:60\n" ] } ], "source": [ "add!(model, Dense(80, input_dim=700))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We need an activation function to transmit signals down to the next layer.\n", "\n", "The activation function of the input layer is Relu. This function is used to \"activate\" nodes in the hidden layer." ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "┌ Warning: `getindex(o::PyObject, s::Symbol)` is deprecated in favor of dot overloading (`getproperty`) so elements should now be accessed as e.g. `o.s` instead of `o[:s]`.\n", "│ caller = (::getfield(Keras.Layers, Symbol(\"##Activation#43#45\")))(::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::Type, ::Symbol) at layers.jl:168\n", "└ @ Keras.Layers C:\\Users\\simon\\.julia\\dev\\Keras\\src\\layers.jl:168\n", "┌ Warning: `getindex(o::PyObject, s::Symbol)` is deprecated in favor of dot overloading (`getproperty`) so elements should now be accessed as e.g. `o.s` instead of `o[:s]`.\n", "│ caller = add!(::Sequential, ::Activation) at models.jl:60\n", "└ @ Keras C:\\Users\\simon\\.julia\\dev\\Keras\\src\\models.jl:60\n" ] } ], "source": [ "add!(model, Activation(:relu))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We set a hidden layer with 10 nodes." ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "┌ Warning: `getindex(o::PyObject, s::Symbol)` is deprecated in favor of dot overloading (`getproperty`) so elements should now be accessed as e.g. `o.s` instead of `o[:s]`.\n", "│ caller = (::getfield(Keras.Layers, Symbol(\"##Dense#40#42\")))(::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::Type, ::Int64) at layers.jl:168\n", "└ @ Keras.Layers C:\\Users\\simon\\.julia\\dev\\Keras\\src\\layers.jl:168\n" ] } ], "source": [ "add!(model, Dense(10))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "The activation function of the hidden layer is softmax. This function is used to \"activate\" nodes in the output layer." ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "add!(model, Activation(:softmax))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Before training a model, we need to configure the learning process, which is done by the compile method. It receives three arguments:\n", "\n", "•\tAn optimizer. This could be the string identifier of an existing optimizer or an instance of the Optimizer class. We use “sgd”, which refers to Stochastic gradient descent optimizer.\n", "\n", "•\tA loss function. This is the objective that the model will try to minimize. Here we use categorical_crossentropy\n", "\n", "•\tA list of metrics. For this classification problem we want to set this to metrics=['accuracy']." ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "┌ Warning: `getindex(o::PyObject, s::Symbol)` is deprecated in favor of dot overloading (`getproperty`) so elements should now be accessed as e.g. `o.s` instead of `o[:s]`.\n", "│ caller = #compile!#8(::Base.Iterators.Pairs{Symbol,Any,Tuple{Symbol,Symbol,Symbol},NamedTuple{(:loss, :optimizer, :metrics),Tuple{Symbol,Symbol,Array{Symbol,1}}}}, ::Function, ::Sequential) at models.jl:64\n", "└ @ Keras C:\\Users\\simon\\.julia\\dev\\Keras\\src\\models.jl:64\n" ] } ], "source": [ "compile!(model; loss=:categorical_crossentropy, optimizer=:sgd, metrics=[:accuracy])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "The fit method is what we trains the model for a fixed number of epochs. \n", "Epochs are iterations on a dataset. We need to specify a fixed batch size for the inputs when we use the fit method. " ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Here we set epochs to 100, which means we use 100 epochs to train the model. \n", "\n", "And the batch_size is 32, which shows the number of samples per gradient update. \n", "\n", "The last parameter is verbose=1, because we want to see the progress bar. If you don’t want see anything, set this to 0." ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "scrolled": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Train on 8 samples, validate on 2 samples\r\n", "Epoch 1/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 13.0253 - acc: 0.0000e+00 - val_loss: 13.3730 - val_acc: 0.0000e+00\r\n", "Epoch 2/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 12.3513 - acc: 0.1250 - val_loss: 13.8142 - val_acc: 0.5000\r\n", "Epoch 3/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 12.0592 - acc: 0.2500 - val_loss: 13.3732 - val_acc: 0.0000e+00\r\n", "Epoch 4/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.9330 - acc: 0.3750 - val_loss: 13.5297 - val_acc: 0.5000\r\n", "Epoch 5/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.8625 - acc: 0.3750 - val_loss: 13.4296 - val_acc: 0.5000\r\n", "Epoch 6/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.8138 - acc: 0.3750 - val_loss: 13.4939 - val_acc: 0.5000\r\n", "Epoch 7/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.7720 - acc: 0.3750 - val_loss: 13.4561 - val_acc: 0.5000\r\n", "Epoch 8/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.7369 - acc: 0.3750 - val_loss: 13.4960 - val_acc: 0.5000\r\n", "Epoch 9/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.7043 - acc: 0.3750 - val_loss: 13.4806 - val_acc: 0.5000\r\n", "Epoch 10/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.6756 - acc: 0.3750 - val_loss: 13.5014 - val_acc: 0.5000\r\n", "Epoch 11/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.6489 - acc: 0.3750 - val_loss: 13.4913 - val_acc: 0.5000\r\n", "Epoch 12/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.6251 - acc: 0.5000 - val_loss: 13.4976 - val_acc: 0.5000\r\n", "Epoch 13/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.6034 - acc: 0.5000 - val_loss: 13.5082 - val_acc: 0.5000\r\n", "Epoch 14/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.5837 - acc: 0.5000 - val_loss: 13.5067 - val_acc: 0.5000\r\n", "Epoch 15/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.5649 - acc: 0.5000 - val_loss: 13.5195 - val_acc: 0.5000\r\n", "Epoch 16/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.5475 - acc: 0.5000 - val_loss: 13.5165 - val_acc: 0.0000e+00\r\n", "Epoch 17/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.5311 - acc: 0.5000 - val_loss: 13.5178 - val_acc: 0.0000e+00\r\n", "Epoch 18/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.5161 - acc: 0.5000 - val_loss: 13.5205 - val_acc: 0.0000e+00\r\n", "Epoch 19/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.5017 - acc: 0.5000 - val_loss: 13.5351 - val_acc: 0.0000e+00\r\n", "Epoch 20/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.4883 - acc: 0.5000 - val_loss: 13.5183 - val_acc: 0.0000e+00\r\n", "Epoch 21/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.4754 - acc: 0.5000 - val_loss: 13.5398 - val_acc: 0.5000\r\n", "Epoch 22/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.4628 - acc: 0.5000 - val_loss: 13.5308 - val_acc: 0.0000e+00\r\n", "Epoch 23/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.4512 - acc: 0.5000 - val_loss: 13.5516 - val_acc: 0.0000e+00\r\n", "Epoch 24/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.4411 - acc: 0.5000 - val_loss: 13.5409 - val_acc: 0.0000e+00\r\n", "Epoch 25/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.4318 - acc: 0.5000 - val_loss: 13.5530 - val_acc: 0.0000e+00\r\n", "Epoch 26/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.4223 - acc: 0.5000 - val_loss: 13.5500 - val_acc: 0.0000e+00\r\n", "Epoch 27/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.4139 - acc: 0.5000 - val_loss: 13.5634 - val_acc: 0.0000e+00\r\n", "Epoch 28/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.4060 - acc: 0.5000 - val_loss: 13.5546 - val_acc: 0.0000e+00\r\n", "Epoch 29/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.3979 - acc: 0.5000 - val_loss: 13.5727 - val_acc: 0.0000e+00\r\n", "Epoch 30/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.3906 - acc: 0.5000 - val_loss: 13.5622 - val_acc: 0.0000e+00\r\n", "Epoch 31/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.3832 - acc: 0.5000 - val_loss: 13.5775 - val_acc: 0.0000e+00\r\n", "Epoch 32/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.3757 - acc: 0.6250 - val_loss: 13.5721 - val_acc: 0.0000e+00\r\n", "Epoch 33/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.3688 - acc: 0.6250 - val_loss: 13.5886 - val_acc: 0.0000e+00\r\n", "Epoch 34/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.3621 - acc: 0.6250 - val_loss: 13.5815 - val_acc: 0.0000e+00\r\n", "Epoch 35/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.3558 - acc: 0.6250 - val_loss: 13.6035 - val_acc: 0.0000e+00\r\n", "Epoch 36/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.3501 - acc: 0.6250 - val_loss: 13.5932 - val_acc: 0.0000e+00\r\n", "Epoch 37/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.3443 - acc: 0.6250 - val_loss: 13.6026 - val_acc: 0.0000e+00\r\n", "Epoch 38/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.3395 - acc: 0.6250 - val_loss: 13.6002 - val_acc: 0.0000e+00\r\n", "Epoch 39/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.3346 - acc: 0.6250 - val_loss: 13.6208 - val_acc: 0.0000e+00\r\n", "Epoch 40/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.3299 - acc: 0.6250 - val_loss: 13.6007 - val_acc: 0.0000e+00\r\n", "Epoch 41/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.3255 - acc: 0.6250 - val_loss: 13.6267 - val_acc: 0.0000e+00\r\n", "Epoch 42/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.3213 - acc: 0.6250 - val_loss: 13.6189 - val_acc: 0.0000e+00\r\n", "Epoch 43/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.3168 - acc: 0.6250 - val_loss: 13.6338 - val_acc: 0.0000e+00\r\n", "Epoch 44/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.3132 - acc: 0.6250 - val_loss: 13.6216 - val_acc: 0.0000e+00\r\n", "Epoch 45/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.3094 - acc: 0.6250 - val_loss: 13.6389 - val_acc: 0.0000e+00\r\n", "Epoch 46/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.3057 - acc: 0.6250 - val_loss: 13.6284 - val_acc: 0.0000e+00\r\n", "Epoch 47/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.3024 - acc: 0.6250 - val_loss: 13.6483 - val_acc: 0.0000e+00\r\n", "Epoch 48/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2988 - acc: 0.6250 - val_loss: 13.6351 - val_acc: 0.0000e+00\r\n", "Epoch 49/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2954 - acc: 0.7500 - val_loss: 13.6499 - val_acc: 0.0000e+00\r\n", "Epoch 50/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2923 - acc: 0.6250 - val_loss: 13.6394 - val_acc: 0.0000e+00\r\n", "Epoch 51/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2896 - acc: 0.6250 - val_loss: 13.6597 - val_acc: 0.0000e+00\r\n", "Epoch 52/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2864 - acc: 0.8750 - val_loss: 13.6443 - val_acc: 0.0000e+00\r\n", "Epoch 53/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2837 - acc: 0.7500 - val_loss: 13.6602 - val_acc: 0.0000e+00\r\n", "Epoch 54/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2809 - acc: 0.6250 - val_loss: 13.6528 - val_acc: 0.0000e+00\r\n", "Epoch 55/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2782 - acc: 0.7500 - val_loss: 13.6665 - val_acc: 0.0000e+00\r\n", "Epoch 56/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2756 - acc: 0.8750 - val_loss: 13.6603 - val_acc: 0.0000e+00\r\n", "Epoch 57/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2733 - acc: 0.7500 - val_loss: 13.6658 - val_acc: 0.0000e+00\r\n", "Epoch 58/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2709 - acc: 0.7500 - val_loss: 13.6687 - val_acc: 0.0000e+00\r\n", "Epoch 59/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2686 - acc: 0.8750 - val_loss: 13.6692 - val_acc: 0.0000e+00\r\n", "Epoch 60/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2664 - acc: 0.8750 - val_loss: 13.6744 - val_acc: 0.0000e+00\r\n", "Epoch 61/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2643 - acc: 0.8750 - val_loss: 13.6743 - val_acc: 0.0000e+00\r\n", "Epoch 62/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2622 - acc: 0.8750 - val_loss: 13.6759 - val_acc: 0.0000e+00\r\n", "Epoch 63/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2601 - acc: 0.8750 - val_loss: 13.6779 - val_acc: 0.0000e+00\r\n", "Epoch 64/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2581 - acc: 0.8750 - val_loss: 13.6796 - val_acc: 0.0000e+00\r\n", "Epoch 65/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2563 - acc: 0.8750 - val_loss: 13.6849 - val_acc: 0.0000e+00\r\n", "Epoch 66/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2544 - acc: 0.8750 - val_loss: 13.6801 - val_acc: 0.0000e+00\r\n", "Epoch 67/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2526 - acc: 0.8750 - val_loss: 13.6869 - val_acc: 0.0000e+00\r\n", "Epoch 68/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2510 - acc: 0.8750 - val_loss: 13.6865 - val_acc: 0.0000e+00\r\n", "Epoch 69/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2492 - acc: 0.8750 - val_loss: 13.6917 - val_acc: 0.0000e+00\r\n", "Epoch 70/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2477 - acc: 0.8750 - val_loss: 13.6881 - val_acc: 0.0000e+00\r\n", "Epoch 71/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2462 - acc: 0.8750 - val_loss: 13.6968 - val_acc: 0.0000e+00\r\n", "Epoch 72/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2447 - acc: 0.8750 - val_loss: 13.6939 - val_acc: 0.0000e+00\r\n", "Epoch 73/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2431 - acc: 0.8750 - val_loss: 13.7007 - val_acc: 0.0000e+00\r\n", "Epoch 74/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2417 - acc: 0.8750 - val_loss: 13.6934 - val_acc: 0.0000e+00\r\n", "Epoch 75/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2404 - acc: 0.8750 - val_loss: 13.7039 - val_acc: 0.0000e+00\r\n", "Epoch 76/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2389 - acc: 0.8750 - val_loss: 13.7018 - val_acc: 0.0000e+00\r\n", "Epoch 77/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2377 - acc: 0.8750 - val_loss: 13.7034 - val_acc: 0.0000e+00\r\n", "Epoch 78/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2365 - acc: 0.8750 - val_loss: 13.6990 - val_acc: 0.0000e+00\r\n", "Epoch 79/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2352 - acc: 0.8750 - val_loss: 13.7102 - val_acc: 0.0000e+00\r\n", "Epoch 80/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2341 - acc: 0.8750 - val_loss: 13.7036 - val_acc: 0.0000e+00\r\n", "Epoch 81/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2329 - acc: 0.8750 - val_loss: 13.7106 - val_acc: 0.0000e+00\r\n", "Epoch 82/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2317 - acc: 0.8750 - val_loss: 13.7084 - val_acc: 0.0000e+00\r\n", "Epoch 83/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2306 - acc: 0.8750 - val_loss: 13.7147 - val_acc: 0.0000e+00\r\n", "Epoch 84/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2296 - acc: 0.8750 - val_loss: 13.7113 - val_acc: 0.0000e+00\r\n", "Epoch 85/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2285 - acc: 0.8750 - val_loss: 13.7165 - val_acc: 0.0000e+00\r\n", "Epoch 86/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2275 - acc: 0.8750 - val_loss: 13.7172 - val_acc: 0.0000e+00\r\n", "Epoch 87/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2265 - acc: 0.8750 - val_loss: 13.7186 - val_acc: 0.0000e+00\r\n", "Epoch 88/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2255 - acc: 0.8750 - val_loss: 13.7198 - val_acc: 0.0000e+00\r\n", "Epoch 89/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2246 - acc: 0.8750 - val_loss: 13.7233 - val_acc: 0.0000e+00\r\n", "Epoch 90/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2237 - acc: 0.8750 - val_loss: 13.7209 - val_acc: 0.0000e+00\r\n", "Epoch 91/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2228 - acc: 0.8750 - val_loss: 13.7283 - val_acc: 0.0000e+00\r\n", "Epoch 92/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2219 - acc: 0.8750 - val_loss: 13.7296 - val_acc: 0.0000e+00\r\n", "Epoch 93/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2210 - acc: 0.8750 - val_loss: 13.7296 - val_acc: 0.0000e+00\r\n", "Epoch 94/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2202 - acc: 0.8750 - val_loss: 13.7321 - val_acc: 0.0000e+00\r\n", "Epoch 95/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2195 - acc: 0.8750 - val_loss: 13.7333 - val_acc: 0.0000e+00\r\n", "Epoch 96/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2186 - acc: 0.8750 - val_loss: 13.7372 - val_acc: 0.0000e+00\r\n", "Epoch 97/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2178 - acc: 0.8750 - val_loss: 13.7377 - val_acc: 0.0000e+00\r\n", "Epoch 98/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2171 - acc: 0.8750 - val_loss: 13.7397 - val_acc: 0.0000e+00\r\n", "Epoch 99/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2163 - acc: 0.8750 - val_loss: 13.7371 - val_acc: 0.0000e+00\r\n", "Epoch 100/100\r\n", "\r", "8/8 [==============================] - 0s - loss: 11.2156 - acc: 0.8750 - val_loss: 13.7437 - val_acc: 0.0000e+00\r\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "┌ Warning: `getindex(o::PyObject, s::Symbol)` is deprecated in favor of dot overloading (`getproperty`) so elements should now be accessed as e.g. `o.s` instead of `o[:s]`.\n", "│ caller = #fit!#9(::Int64, ::Base.Iterators.Pairs{Symbol,Real,Tuple{Symbol,Symbol,Symbol},NamedTuple{(:validation_split, :nb_epoch, :batch_size),Tuple{Float64,Int64,Int64}}}, ::Function, ::Sequential, ::Array{Float64,2}, ::Vararg{Array{Float64,2},N} where N) at models.jl:68\n", "└ @ Keras C:\\Users\\simon\\.julia\\dev\\Keras\\src\\models.jl:68\n", "C:\\Users\\simon\\.julia\\conda\\3\\lib\\site-packages\\keras\\models.py:826: UserWarning: The `nb_epoch` argument in `fit` has been renamed `epochs`.\r\n", " warnings.warn('The `nb_epoch` argument in `fit` '\r\n" ] }, { "data": { "text/plain": [ "PyObject " ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "h = fit!(model, rand(10, 700), rand(10, 10); validation_split=0.20, nb_epoch=100, batch_size=32, verbose=1)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Overfitting always occurs when our model becomes really good at classifying or predicting on data that was included in the training set. But it is not good at classifying data that isn't trained on.\n", "\n", "We ca know overfitting based on metrics that are given for our training and validation data during the training process.\n", "\n", "If the validation accuracy worse than training accuracy, then our model is overfitting.\n", "\n", "We also get overfitting if the models metrics were good but when we use the model to predict on test data, it is not accurately classifying the data in the test set." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "In the test mode, we need input data and target data, and the calculation is carried out in batches.\n", "\n", "Then it returns to the error value (loss) and the evaluation standard value.\n", "\n", "We can get the evaluation standard value is about 0.2, which is small and close to 0." ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "┌ Warning: `getindex(o::PyObject, s::Symbol)` is deprecated in favor of dot overloading (`getproperty`) so elements should now be accessed as e.g. `o.s` instead of `o[:s]`.\n", "│ caller = #evaluate#10(::Int64, ::Int64, ::Function, ::Sequential, ::Array{Float64,2}, ::Array{Float64,2}) at models.jl:72\n", "└ @ Keras C:\\Users\\simon\\.julia\\dev\\Keras\\src\\models.jl:72\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\r", " 5/10 [==============>...............] - ETA: 0s" ] }, { "data": { "text/plain": [ "2-element Array{Float64,1}:\n", " 11.976428508758545\n", " 0.0 " ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "evaluate(model, rand(10, 700), rand(10, 10); batch_size=5, verbose=1)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Then we can use this model to predict on other dataset." ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\r", " 5/10 [==============>...............] - ETA: 0s" ] }, { "name": "stderr", "output_type": "stream", "text": [ "┌ Warning: `getindex(o::PyObject, s::Symbol)` is deprecated in favor of dot overloading (`getproperty`) so elements should now be accessed as e.g. `o.s` instead of `o[:s]`.\n", "│ caller = #predict#11(::Int64, ::Int64, ::Function, ::Sequential, ::Array{Float64,2}) at models.jl:76\n", "└ @ Keras C:\\Users\\simon\\.julia\\dev\\Keras\\src\\models.jl:76\n" ] }, { "data": { "text/plain": [ "10×10 Array{Float32,2}:\n", " 0.0987271 0.133345 0.118539 … 0.0968778 0.0635645 0.0530889\n", " 0.0641723 0.0983444 0.0940889 0.172037 0.107849 0.0709795\n", " 0.0876788 0.0875767 0.0986511 0.168873 0.0646441 0.0912005\n", " 0.138393 0.124779 0.0870739 0.110914 0.135138 0.136531 \n", " 0.0708688 0.0960495 0.0594764 0.119417 0.0838852 0.142463 \n", " 0.0663076 0.105571 0.0610621 … 0.0827518 0.157182 0.072068 \n", " 0.106416 0.100854 0.0843876 0.0752178 0.105821 0.0441032\n", " 0.0711867 0.162014 0.0966662 0.0492741 0.207805 0.0866259\n", " 0.0878673 0.165861 0.142253 0.0647497 0.159111 0.0748197\n", " 0.0566073 0.064067 0.09489 0.124726 0.0966054 0.0993727" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "predict(model, rand(10, 700); batch_size=5, verbose=1)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Reference:\n", "\n", "https://keras.io/models/\n", "\n", "https://github.com/invenia/Keras.jl" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Youtube Link:\n", "\n", "https://youtu.be/9_ILdjXJu8E" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Julia 1.1.1", "language": "julia", "name": "julia-1.1" }, "language_info": { "file_extension": ".jl", "mimetype": "application/julia", "name": "julia", "version": "1.1.1" } }, "nbformat": 4, "nbformat_minor": 2 }