GNUnet  0.10.x
plugin_ats_ril.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet.
3  Copyright (C) 2011-2014 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19  */
20 
27 #include "platform.h"
28 #include <float.h>
29 #include <math.h>
30 #include "gnunet_ats_plugin.h"
32 
33 
34 
35 #define LOG(kind, ...) GNUNET_log_from (kind, "ats-ril", __VA_ARGS__)
36 
37 #define RIL_MIN_BW (5 * ntohl ( \
38  GNUNET_CONSTANTS_DEFAULT_BW_IN_OUT. \
39  value__))
40 #define RIL_MAX_BW GNUNET_ATS_MaxBandwidth
41 
42 #define RIL_ACTION_INVALID -1
43 #define RIL_INTERVAL_EXPONENT 10
44 #define RIL_UTILITY_DELAY_MAX 1000
45 
46 #define RIL_DEFAULT_STEP_TIME_MIN GNUNET_TIME_relative_multiply ( \
47  GNUNET_TIME_UNIT_MILLISECONDS, 200)
48 #define RIL_DEFAULT_STEP_TIME_MAX GNUNET_TIME_relative_multiply ( \
49  GNUNET_TIME_UNIT_MILLISECONDS, 2000)
50 #define RIL_DEFAULT_ALGORITHM RIL_ALGO_SARSA
51 #define RIL_DEFAULT_SELECT RIL_SELECT_SOFTMAX
52 #define RIL_DEFAULT_WELFARE RIL_WELFARE_NASH
53 #define RIL_DEFAULT_DISCOUNT_BETA 0.6
54 #define RIL_DEFAULT_DISCOUNT_GAMMA 0.5
55 #define RIL_DEFAULT_GRADIENT_STEP_SIZE 0.01
56 #define RIL_DEFAULT_TRACE_DECAY 0.5
57 #define RIL_DEFAULT_EXPLORE_RATIO 1
58 #define RIL_DEFAULT_EXPLORE_DECAY 0.95
59 #define RIL_DEFAULT_RBF_DIVISOR 50
60 #define RIL_DEFAULT_TEMPERATURE 0.1
61 #define RIL_DEFAULT_TEMPERATURE_DECAY 1
62 
63 #define RIL_INC_DEC_STEP_SIZE 1
64 #define RIL_NOP_DECAY 0.5
65 
79 {
81  RIL_ACTION_BW_IN_DBL = -2, // TODO? Potentially add more actions
90 };
91 
93 {
96 };
97 
99 {
102 };
103 
105 {
108 };
109 
111 {
116 };
117 
122 {
127 
131  double alpha;
132 
136  double beta;
137 
141  double gamma;
142 
146  double lambda;
147 
152 
157 
161  double temperature;
162 
167 
172 
176  unsigned long long rbf_divisor;
177 
182 
186  double epsilon_init;
187 
191  double epsilon;
192 
197 
202 
207 };
208 
213 {
218 
223 
228 };
229 
230 
232 {
237 
242 
247 
252 
257 
261  unsigned long long step_count;
262 
266  double **W;
267 
271  unsigned int m;
272 
276  unsigned int n;
277 
281  double *s_old;
282 
286  int a_old;
287 
291  double **E;
292 
297 
302 
307 
312 
316  uint32_t bw_in;
317 
321  uint32_t bw_out;
322 
327 
332 
337 
341  double nop_bonus;
342 };
343 
344 struct RIL_Scope
345 {
350 
354  uint32_t bw_in_available;
355 
359  uint32_t bw_in_assigned;
360 
364  uint32_t bw_in_utilized;
365 
370 
374  unsigned long long bw_out_assigned;
375 
379  unsigned long long bw_out_utilized;
380 
384  unsigned int active_agent_count;
385 
390 };
391 
396 {
401 
405  unsigned long long step_count;
406 
410  struct GNUNET_TIME_Absolute step_time_last;
411 
416 
421 
426 
431 
436 
440  struct RIL_Learning_Parameters parameters;
441 
446 
450  unsigned int networks_count;
451 
457 
461  int done;
462 
466  unsigned long long simulate;
467 };
468 
469 /*
470  * "Private" functions
471  * ---------------------------
472  */
473 
482 static double
483 agent_q (struct RIL_Peer_Agent *agent,
484  const double *state,
485  int action)
486 {
487  unsigned int i;
488  double result = 0.0;
489 
490  for (i = 0; i < agent->m; i++)
491  result += state[i] * agent->W[action][i];
492 
493  /* prevent crashes if learning diverges */
494  if (isnan (result))
495  return isnan (result) * UINT32_MAX;
496  if (isinf (result))
497  return isinf (result) * UINT32_MAX;
498  return result;
499 }
500 
501 
509 static int
512 {
513  int i;
514  struct RIL_Address_Wrapped *cur;
515 
516  i = -1;
517  for (cur = agent->addresses_head; NULL != cur; cur = cur->next)
518  {
519  i++;
520  if (cur->address_naked == address)
521  return i;
522  }
523  return i;
524 }
525 
526 
534 static struct RIL_Address_Wrapped *
537 {
538  struct RIL_Address_Wrapped *cur;
539 
540  for (cur = agent->addresses_head; NULL != cur; cur = cur->next)
541  if (cur->address_naked == address)
542  return cur;
543  return NULL;
544 }
545 
546 
547 static int
548 agent_action_is_possible (struct RIL_Peer_Agent *agent, int action)
549 {
550  int address_index;
551 
552  switch (action)
553  {
554  case RIL_ACTION_NOTHING:
555  return GNUNET_YES;
556  break;
557 
560  if (agent->bw_in >= RIL_MAX_BW)
561  return GNUNET_NO;
562  else
563  return GNUNET_YES;
564  break;
565 
568  if (agent->bw_in <= 0)
569  return GNUNET_NO;
570  else
571  return GNUNET_YES;
572  break;
573 
576  if (agent->bw_out >= RIL_MAX_BW)
577  return GNUNET_NO;
578  else
579  return GNUNET_YES;
580  break;
581 
584  if (agent->bw_out <= 0)
585  return GNUNET_NO;
586  else
587  return GNUNET_YES;
588  break;
589 
590  default:
591  if ((action >= RIL_ACTION_TYPE_NUM) && (action < agent->n)) // switch address action
592  {
593  address_index = action - RIL_ACTION_TYPE_NUM;
594 
595  GNUNET_assert (address_index >= 0);
596  GNUNET_assert (
597  address_index <= agent_address_get_index (agent,
598  agent->addresses_tail->
599  address_naked));
600 
601  if ((agent_address_get_index (agent, agent->address_inuse) ==
602  address_index) ||
603  agent->address_inuse->active)
604  return GNUNET_NO;
605  else
606  return GNUNET_YES;
607  break;
608  }
609  // error - action does not exist
611  }
612 }
613 
614 
623 static int
624 agent_get_action_max (struct RIL_Peer_Agent *agent, double *state)
625 {
626  int i;
627  int max_i = RIL_ACTION_INVALID;
628  double cur_q;
629  double max_q = -DBL_MAX;
630 
631  for (i = 0; i < agent->n; i++)
632  {
633  if (agent_action_is_possible (agent, i))
634  {
635  cur_q = agent_q (agent, state, i);
636  if (cur_q > max_q)
637  {
638  max_q = cur_q;
639  max_i = i;
640  }
641  }
642  }
643 
645 
646  return max_i;
647 }
648 
655 static int
657 {
658  int i;
659  int is_possible[agent->n];
660  int sum = 0;
661  int r;
662 
663  for (i = 0; i < agent->n; i++)
664  {
665  if (agent_action_is_possible (agent, i))
666  {
667  is_possible[i] = GNUNET_YES;
668  sum++;
669  }
670  else
671  {
672  is_possible[i] = GNUNET_NO;
673  }
674  }
675 
677 
678  sum = -1;
679  for (i = 0; i < agent->n; i++)
680  {
681  if (is_possible[i])
682  {
683  sum++;
684  if (sum == r)
685  return i;
686  }
687  }
688 
690  return RIL_ACTION_INVALID;
691 }
692 
693 
702 static void
703 agent_update (struct RIL_Peer_Agent *agent, double reward, double *s_next, int
704  a_prime)
705 {
706  int i;
707  int k;
708  double delta;
709  double **theta = agent->W;
710 
711  delta = agent->envi->global_discount_integrated * reward; // reward
712  delta += agent->envi->global_discount_variable * agent_q (agent, s_next,
713  a_prime); // discounted future value
714  delta -= agent_q (agent, agent->s_old, agent->a_old); // one step
715 
716 // LOG(GNUNET_ERROR_TYPE_INFO, "update() Step# %llu Q(s,a): %f a: %f r: %f y: %f Q(s+1,a+1) = %f delta: %f\n",
717 // agent->step_count,
718 // agent_q (agent, agent->s_old, agent->a_old),
719 // agent->envi->parameters.alpha,
720 // reward,
721 // agent->envi->global_discount_variable,
722 // agent_q (agent, s_next, a_prime),
723 // delta);
724 
725  for (k = 0; k < agent->n; k++)
726  {
727  for (i = 0; i < agent->m; i++)
728  {
729  // LOG(GNUNET_ERROR_TYPE_INFO, "alpha = %f delta = %f e[%d] = %f\n",
730  // agent->envi->parameters.alpha,
731  // delta,
732  // i,
733  // agent->e[i]);
734  theta[k][i] += agent->envi->parameters.alpha * delta * agent->E[k][i];
735  }
736  }
737 }
738 
739 
752 static void
754  enum RIL_E_Modification mod,
755  double *feature,
756  int action)
757 {
758  int i;
759  int k;
760 
761  for (i = 0; i < agent->m; i++)
762  {
763  switch (mod)
764  {
765  case RIL_E_ACCUMULATE:
766  agent->E[action][i] += feature[i];
767  break;
768 
769  case RIL_E_REPLACE:
770  agent->E[action][i] = agent->E[action][i] > feature[i] ?
771  agent->E[action][i] : feature[i];
772  break;
773 
774  case RIL_E_DECAY:
775  for (k = 0; k < agent->n; k++)
776  {
777  agent->E[k][i] *= agent->envi->global_discount_variable
778  * agent->envi->parameters.lambda;
779  }
780  break;
781 
782  case RIL_E_ZERO:
783  for (k = 0; k < agent->n; k++)
784  {
785  agent->E[k][i] = 0;
786  }
787  break;
788  }
789  }
790 }
791 
799 static void
800 ril_inform (struct GAS_RIL_Handle *solver,
802  enum GAS_Solver_Status stat)
803 {
804  solver->env->info_cb (solver->env->cls,
805  op,
806  stat,
807  GAS_INFO_NONE);
808 }
809 
815 static unsigned long long
817 {
818  return GNUNET_MIN (2 * GNUNET_MAX (net->bw_in_available,
819  net->bw_out_available),
821 }
822 
834 static void
836  struct RIL_Peer_Agent *agent,
837  struct ATS_Address *new_address,
838  unsigned long long new_bw_in,
839  unsigned long long new_bw_out,
840  int silent)
841 {
842  int notify = GNUNET_NO;
843 
845  " set_active_suggestion() for peer '%s'\n",
846  GNUNET_i2s (&agent->peer));
847 
848  // address change
849  if (agent->address_inuse != new_address)
850  {
851  if (NULL != agent->address_inuse)
852  {
853  agent->address_inuse->active = GNUNET_NO;
854  agent->address_inuse->assigned_bw_in = 0;
855  agent->address_inuse->assigned_bw_out = 0;
856  }
857  if (NULL != new_address)
858  {
859  LOG (GNUNET_ERROR_TYPE_DEBUG, " set address active: %s\n",
860  agent->is_active ? "yes" : "no");
861  new_address->active = agent->is_active;
862  new_address->assigned_bw_in = agent->bw_in;
863  new_address->assigned_bw_out = agent->bw_out;
864  }
865  notify |= GNUNET_YES;
866  }
867 
868  if (new_address)
869  {
870  // activity change
871  if (new_address->active != agent->is_active)
872  {
873  new_address->active = agent->is_active;
874  notify |= GNUNET_YES;
875  }
876 
877  // bw change
878  if (agent->bw_in != new_bw_in)
879  {
880  agent->bw_in = new_bw_in;
881  new_address->assigned_bw_in = new_bw_in;
882  notify |= GNUNET_YES;
883  }
884  if (agent->bw_out != new_bw_out)
885  {
886  agent->bw_out = new_bw_out;
887  new_address->assigned_bw_out = new_bw_out;
888  notify |= GNUNET_YES;
889  }
890  }
891 
892  if (notify && agent->is_active && (GNUNET_NO == silent))
893  {
894  if (new_address)
895  {
897  " envi_set_active_suggestion() notify\n");
898  agent->suggestion_issue = GNUNET_YES;
899  agent->suggestion_address = new_address;
900  }
901  else if (agent->address_inuse)
902  {
903  /* disconnect case, no new address */
906  agent->bw_in = 0;
907  agent->bw_out = 0;
908 
909  agent->suggestion_issue = GNUNET_YES;
910  agent->suggestion_address = agent->address_inuse;
911  }
912  }
913  agent->address_inuse = new_address;
914 }
915 
916 
923 static double *
924 envi_get_state (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent)
925 {
926  double *state;
927  double y[2];
928  double x[2];
929  double d[2];
930  double sigma;
931  double f;
932  int m;
933  int i;
934  int k;
935  unsigned long long max_bw;
936 
937  state = GNUNET_malloc (sizeof(double) * agent->m);
938 
939  max_bw = ril_get_max_bw ((struct
940  RIL_Scope *) agent->address_inuse->
941  solver_information);
942 
943  y[0] = (double) agent->bw_out;
944  y[1] = (double) agent->bw_in;
945 
946  m = agent_address_get_index (agent, agent->address_inuse)
947  * (solver->parameters.rbf_divisor + 1) * (solver->parameters.rbf_divisor
948  + 1);
949  for (i = 0; i <= solver->parameters.rbf_divisor; i++)
950  {
951  for (k = 0; k <= solver->parameters.rbf_divisor; k++)
952  {
953  x[0] = (double) i * (double) max_bw
954  / (double) solver->parameters.rbf_divisor;
955  x[1] = (double) k * (double) max_bw
956  / (double) solver->parameters.rbf_divisor;
957  d[0] = x[0] - y[0];
958  d[1] = x[1] - y[1];
959  sigma = (((double) max_bw / ((double) solver->parameters.rbf_divisor
960  + 1)) * 0.5);
961  f = exp (-((d[0] * d[0] + d[1] * d[1]) / (2 * sigma * sigma)));
962  state[m++] = f;
963  }
964  }
965 
966  return state;
967 }
968 
969 
976 static double
978 {
979  const double *preferences;
980  double delay_atsi;
981  double delay_norm;
982  double pref_match;
983 
984  preferences = agent->envi->env->get_preferences (agent->envi->env->cls,
985  &agent->peer);
986 
987  delay_atsi = agent->address_inuse->norm_delay.norm;
988  delay_norm = RIL_UTILITY_DELAY_MAX * exp (-delay_atsi * 0.00001);
989 
990  pref_match = preferences[GNUNET_ATS_PREFERENCE_LATENCY] * delay_norm;
991  pref_match += preferences[GNUNET_ATS_PREFERENCE_BANDWIDTH]
992  * sqrt ((double) (agent->bw_in / RIL_MIN_BW)
993  * (double) (agent->bw_out / RIL_MIN_BW));
994  return pref_match;
995 }
996 
1005 static double
1007  RIL_Scope *scope)
1008 {
1009  struct RIL_Peer_Agent *cur;
1010  double result;
1011 
1012  switch (solver->parameters.social_welfare)
1013  {
1015  result = DBL_MAX;
1016  for (cur = solver->agents_head; NULL != cur; cur = cur->next)
1017  {
1018  if (cur->is_active && cur->address_inuse &&
1019  (cur->address_inuse->solver_information == scope))
1020  {
1021  result = GNUNET_MIN (result, agent_get_utility (cur));
1022  }
1023  }
1024  return result;
1025 
1026  case RIL_WELFARE_NASH:
1027  result = 0;
1028  for (cur = solver->agents_head; NULL != cur; cur = cur->next)
1029  {
1030  if (cur->is_active && cur->address_inuse &&
1031  (cur->address_inuse->solver_information == scope))
1032  {
1033  result *= pow (agent_get_utility (cur), 1.0
1034  / (double) scope->active_agent_count);
1035  }
1036  }
1037  return result;
1038  }
1040  return 1;
1041 }
1042 
1043 static double
1044 envi_get_penalty (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent)
1045 {
1046  struct RIL_Scope *net;
1047  unsigned long long over_max;
1048  unsigned long long over_in = 0;
1049  unsigned long long over_out = 0;
1050 
1051  net = agent->address_inuse->solver_information;
1052 
1053  if (net->bw_in_utilized > net->bw_in_available)
1054  {
1055  over_in = net->bw_in_utilized - net->bw_in_available;
1056  if (RIL_ACTION_BW_IN_INC == agent->a_old)
1057  {
1058  /* increase quadratically */
1059  over_in *= over_in;
1060  }
1061  }
1062  if (net->bw_out_utilized > net->bw_out_available)
1063  {
1064  over_out = net->bw_out_utilized - net->bw_out_available;
1065  if (RIL_ACTION_BW_OUT_INC == agent->a_old)
1066  {
1067  /* increase quadratically */
1068  over_out *= over_out;
1069  }
1070  }
1071  over_max = (over_in + over_out) / (RIL_MIN_BW * RIL_MIN_BW);
1072 
1073  return -1.0 * (double) over_max;
1074 }
1075 
1085 static double
1086 envi_get_reward (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent)
1087 {
1088  struct RIL_Scope *net;
1089  double objective;
1090  double delta;
1091  double steady;
1092  double penalty;
1093  double reward;
1094 
1095  net = agent->address_inuse->solver_information;
1096 
1097  penalty = envi_get_penalty (solver, agent);
1098  objective = (agent_get_utility (agent) + net->social_welfare) / 2;
1099  delta = objective - agent->objective_old;
1100  agent->objective_old = objective;
1101 
1102  if ((delta != 0)&&(penalty == 0))
1103  {
1104  agent->nop_bonus = delta * RIL_NOP_DECAY;
1105  }
1106  else
1107  {
1108  agent->nop_bonus *= RIL_NOP_DECAY;
1109  }
1110 
1111  steady = (RIL_ACTION_NOTHING == agent->a_old) ? agent->nop_bonus : 0;
1112 
1113  reward = delta + steady;
1114  return reward + penalty;
1115 }
1116 
1124 static void
1126  struct RIL_Peer_Agent *agent,
1127  int direction_in)
1128 {
1129  unsigned long long new_bw;
1130  unsigned long long max_bw;
1131 
1132  max_bw = ril_get_max_bw ((struct
1133  RIL_Scope *) agent->address_inuse->
1134  solver_information);
1135 
1136  if (direction_in)
1137  {
1138  new_bw = agent->bw_in * 2;
1139  if ((new_bw < agent->bw_in) ||(new_bw > max_bw) )
1140  new_bw = max_bw;
1141  envi_set_active_suggestion (solver, agent, agent->address_inuse, new_bw,
1142  agent->bw_out, GNUNET_NO);
1143  }
1144  else
1145  {
1146  new_bw = agent->bw_out * 2;
1147  if ((new_bw < agent->bw_out) ||(new_bw > max_bw) )
1148  new_bw = max_bw;
1149  envi_set_active_suggestion (solver, agent, agent->address_inuse,
1150  agent->bw_in,
1151  new_bw, GNUNET_NO);
1152  }
1153 }
1154 
1164 static void
1166  struct RIL_Peer_Agent *agent,
1167  int direction_in)
1168 {
1169  unsigned long long new_bw;
1170 
1171  if (direction_in)
1172  {
1173  new_bw = agent->bw_in / 2;
1174  if ((new_bw <= 0) ||(new_bw > agent->bw_in) )
1175  new_bw = 0;
1176  envi_set_active_suggestion (solver, agent, agent->address_inuse, new_bw,
1177  agent->bw_out,
1178  GNUNET_NO);
1179  }
1180  else
1181  {
1182  new_bw = agent->bw_out / 2;
1183  if ((new_bw <= 0) ||(new_bw > agent->bw_out) )
1184  new_bw = 0;
1185  envi_set_active_suggestion (solver, agent, agent->address_inuse,
1186  agent->bw_in, new_bw,
1187  GNUNET_NO);
1188  }
1189 }
1190 
1199 static void
1200 envi_action_bw_inc (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent,
1201  int direction_in)
1202 {
1203  unsigned long long new_bw;
1204  unsigned long long max_bw;
1205 
1206  max_bw = ril_get_max_bw ((struct
1207  RIL_Scope *) agent->address_inuse->
1208  solver_information);
1209 
1210  if (direction_in)
1211  {
1212  new_bw = agent->bw_in + (RIL_INC_DEC_STEP_SIZE * RIL_MIN_BW);
1213  if ((new_bw < agent->bw_in) ||(new_bw > max_bw) )
1214  new_bw = max_bw;
1215  envi_set_active_suggestion (solver, agent, agent->address_inuse, new_bw,
1216  agent->bw_out, GNUNET_NO);
1217  }
1218  else
1219  {
1220  new_bw = agent->bw_out + (RIL_INC_DEC_STEP_SIZE * RIL_MIN_BW);
1221  if ((new_bw < agent->bw_out) ||(new_bw > max_bw) )
1222  new_bw = max_bw;
1223  envi_set_active_suggestion (solver, agent, agent->address_inuse,
1224  agent->bw_in,
1225  new_bw, GNUNET_NO);
1226  }
1227 }
1228 
1238 static void
1239 envi_action_bw_dec (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent,
1240  int direction_in)
1241 {
1242  unsigned long long new_bw;
1243 
1244  if (direction_in)
1245  {
1246  new_bw = agent->bw_in - (RIL_INC_DEC_STEP_SIZE * RIL_MIN_BW);
1247  if ((new_bw <= 0) ||(new_bw > agent->bw_in) )
1248  new_bw = 0;
1249  envi_set_active_suggestion (solver, agent, agent->address_inuse, new_bw,
1250  agent->bw_out,
1251  GNUNET_NO);
1252  }
1253  else
1254  {
1255  new_bw = agent->bw_out - (RIL_INC_DEC_STEP_SIZE * RIL_MIN_BW);
1256  if ((new_bw <= 0) ||(new_bw > agent->bw_out) )
1257  new_bw = 0;
1258  envi_set_active_suggestion (solver, agent, agent->address_inuse,
1259  agent->bw_in, new_bw,
1260  GNUNET_NO);
1261  }
1262 }
1263 
1271 static void
1273  struct RIL_Peer_Agent *agent,
1274  unsigned int address_index)
1275 {
1276  struct RIL_Address_Wrapped *cur;
1277  int i = 0;
1278 
1279  // cur = agent_address_get_wrapped(agent, agent->address_inuse);
1280 
1281  for (cur = agent->addresses_head; NULL != cur; cur = cur->next)
1282  {
1283  if (i == address_index)
1284  {
1285  envi_set_active_suggestion (solver, agent, cur->address_naked,
1286  agent->bw_in, agent->bw_out,
1287  GNUNET_NO);
1288  return;
1289  }
1290 
1291  i++;
1292  }
1293 
1294  // no address with address_index exists, in this case this action should not be callable
1296 }
1297 
1305 static void
1306 envi_do_action (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int
1307  action)
1308 {
1309  int address_index;
1310 
1311  switch (action)
1312  {
1313  case RIL_ACTION_NOTHING:
1314  break;
1315 
1316  case RIL_ACTION_BW_IN_DBL:
1317  envi_action_bw_double (solver, agent, GNUNET_YES);
1318  break;
1319 
1320  case RIL_ACTION_BW_IN_HLV:
1321  envi_action_bw_halven (solver, agent, GNUNET_YES);
1322  break;
1323 
1324  case RIL_ACTION_BW_IN_INC:
1325  envi_action_bw_inc (solver, agent, GNUNET_YES);
1326  break;
1327 
1328  case RIL_ACTION_BW_IN_DEC:
1329  envi_action_bw_dec (solver, agent, GNUNET_YES);
1330  break;
1331 
1332  case RIL_ACTION_BW_OUT_DBL:
1333  envi_action_bw_double (solver, agent, GNUNET_NO);
1334  break;
1335 
1336  case RIL_ACTION_BW_OUT_HLV:
1337  envi_action_bw_halven (solver, agent, GNUNET_NO);
1338  break;
1339 
1340  case RIL_ACTION_BW_OUT_INC:
1341  envi_action_bw_inc (solver, agent, GNUNET_NO);
1342  break;
1343 
1344  case RIL_ACTION_BW_OUT_DEC:
1345  envi_action_bw_dec (solver, agent, GNUNET_NO);
1346  break;
1347 
1348  default:
1349  if ((action >= RIL_ACTION_TYPE_NUM) && (action < agent->n)) // switch address action
1350  {
1351  address_index = action - RIL_ACTION_TYPE_NUM;
1352 
1353  GNUNET_assert (address_index >= 0);
1354  GNUNET_assert (
1355  address_index <= agent_address_get_index (agent,
1356  agent->addresses_tail->
1357  address_naked));
1358 
1359  envi_action_address_switch (solver, agent, address_index);
1360  break;
1361  }
1362  // error - action does not exist
1364  }
1365 }
1366 
1378 static int
1380 {
1381  int action;
1383  UINT32_MAX)
1384  / (double) UINT32_MAX;
1385 
1386  if (r < agent->envi->parameters.epsilon) // explore
1387  {
1388  action = agent_get_action_random (agent);
1389  if (RIL_ALGO_Q == agent->envi->parameters.algorithm)
1390  {
1391  agent->eligibility_reset = GNUNET_YES;
1392  }
1393  agent->envi->parameters.epsilon *= agent->envi->parameters.epsilon_decay;
1394  return action;
1395  }
1396  else // exploit
1397  {
1398  action = agent_get_action_max (agent, state);
1399  return action;
1400  }
1401 }
1402 
1413 static int
1415 {
1416  int i;
1417  int a_max;
1418  double eqt[agent->n];
1419  double p[agent->n];
1420  double sum = 0;
1421  double r;
1422 
1423  a_max = agent_get_action_max (agent, state);
1424 
1425  for (i = 0; i < agent->n; i++)
1426  {
1427  if (agent_action_is_possible (agent, i))
1428  {
1429  eqt[i] = exp (agent_q (agent, state, i)
1430  / agent->envi->parameters.temperature);
1431  if (isinf (eqt[i]))
1432  eqt[i] = isinf (eqt[i]) * UINT32_MAX;
1433  sum += eqt[i];
1434  }
1435  }
1436  for (i = 0; i < agent->n; i++)
1437  {
1438  if (agent_action_is_possible (agent, i))
1439  {
1440  p[i] = eqt[i] / sum;
1441  }
1442  else
1443  {
1444  p[i] = 0;
1445  }
1446  }
1448  UINT32_MAX) / (double) UINT32_MAX;
1449  sum = 0;
1450  for (i = 0; i < agent->n; i++)
1451  {
1452  if (sum + p[i] > r)
1453  {
1454  if (i != a_max)
1455  {
1456  if (RIL_ALGO_Q == agent->envi->parameters.algorithm)
1457  agent->eligibility_reset = GNUNET_YES;
1458  agent->envi->parameters.temperature *=
1460  }
1461  return i;
1462  }
1463  sum += p[i];
1464  }
1466  return RIL_ACTION_INVALID;
1467 }
1468 
1477 static int
1478 agent_select_action (struct RIL_Peer_Agent *agent, double *state)
1479 {
1480  if (agent->envi->parameters.select == RIL_SELECT_EGREEDY)
1481  {
1482  return agent_select_egreedy (agent, state);
1483  }
1484  else
1485  {
1486  return agent_select_softmax (agent, state);
1487  }
1488 }
1489 
1498 static void
1500 {
1501  int a_next = RIL_ACTION_INVALID;
1502  int a_max;
1503  double *s_next;
1504  double reward;
1505 
1506  LOG (GNUNET_ERROR_TYPE_DEBUG, " agent_step() Peer '%s', algorithm %s\n",
1507  GNUNET_i2s (&agent->peer),
1508  agent->envi->parameters.algorithm ? "Q" : "SARSA");
1509 
1510  s_next = envi_get_state (agent->envi, agent);
1511  reward = envi_get_reward (agent->envi, agent);
1512 
1513  if (agent->eligibility_reset)
1514  {
1515  agent_modify_eligibility (agent, RIL_E_ZERO, NULL, -1);
1516  agent->eligibility_reset = GNUNET_NO;
1517  }
1518  else
1519  {
1520  agent_modify_eligibility (agent, RIL_E_DECAY, NULL, -1);
1521  }
1522  if (RIL_ACTION_INVALID != agent->a_old)
1523  {
1524  agent_modify_eligibility (agent,
1526  agent->s_old, agent->a_old);
1527  }
1528 
1529  switch (agent->envi->parameters.algorithm)
1530  {
1531  case RIL_ALGO_SARSA:
1532  a_next = agent_select_action (agent, s_next);
1533  if (RIL_ACTION_INVALID != agent->a_old)
1534  {
1535  // updates weights with selected action (on-policy), if not first step
1536  agent_update (agent, reward, s_next, a_next);
1537  }
1538  break;
1539 
1540  case RIL_ALGO_Q:
1541  a_max = agent_get_action_max (agent, s_next);
1542  if (RIL_ACTION_INVALID != agent->a_old)
1543  {
1544  // updates weights with best action, disregarding actually selected action (off-policy), if not first step
1545  agent_update (agent, reward, s_next, a_max);
1546  }
1547  a_next = agent_select_action (agent, s_next);
1548  break;
1549  }
1550 
1551  GNUNET_assert (RIL_ACTION_INVALID != a_next);
1552 
1554  "step() Step# %llu R: %f IN %llu OUT %llu A: %d\n",
1555  agent->step_count,
1556  reward,
1557  agent->bw_in / 1024,
1558  agent->bw_out / 1024,
1559  a_next);
1560 
1561  envi_do_action (agent->envi, agent, a_next);
1562 
1563  GNUNET_free (agent->s_old);
1564  agent->s_old = s_next;
1565  agent->a_old = a_next;
1566 
1567  agent->step_count += 1;
1568 }
1569 
1575 static void
1576 ril_step (struct GAS_RIL_Handle *solver);
1577 
1578 
1585 static void
1587 {
1588  struct GAS_RIL_Handle *solver = cls;
1589 
1590  solver->step_next_task_id = NULL;
1591  ril_step (solver);
1592 }
1593 
1602 static double
1604 {
1605  int i;
1606  struct RIL_Scope net;
1607  unsigned long long sum_assigned = 0;
1608  unsigned long long sum_available = 0;
1609  double ratio;
1610 
1611  for (i = 0; i < solver->networks_count; i++)
1612  {
1613  net = solver->network_entries[i];
1614  if (net.bw_in_assigned > 0) // only consider scopes where an address is actually active
1615  {
1616  sum_assigned += net.bw_in_utilized;
1617  sum_assigned += net.bw_out_utilized;
1618  sum_available += net.bw_in_available;
1619  sum_available += net.bw_out_available;
1620  }
1621  }
1622  if (sum_available > 0)
1623  {
1624  ratio = ((double) sum_assigned) / ((double) sum_available);
1625  }
1626  else
1627  {
1628  ratio = 0;
1629  }
1630 
1631  return ratio > 1 ? 1 : ratio; // overutilization is possible, cap at 1
1632 }
1633 
1641 static struct RIL_Scope *
1642 ril_get_network (struct GAS_RIL_Handle *s, uint32_t type)
1643 {
1644  int i;
1645 
1646  for (i = 0; i < s->networks_count; i++)
1647  {
1648  if (s->network_entries[i].type == type)
1649  {
1650  return &s->network_entries[i];
1651  }
1652  }
1653  return NULL;
1654 }
1655 
1665 static int
1667  network)
1668 {
1669  struct RIL_Scope *net;
1670  struct RIL_Peer_Agent *agent;
1671  unsigned long long address_count = 0;
1672 
1673  for (agent = solver->agents_head; NULL != agent; agent = agent->next)
1674  {
1675  if (agent->address_inuse && agent->is_active)
1676  {
1677  net = agent->address_inuse->solver_information;
1678  if (net->type == network)
1679  {
1680  address_count++;
1681  }
1682  }
1683  }
1684 
1685  net = ril_get_network (solver, network);
1686  return (net->bw_in_available > RIL_MIN_BW * address_count) &&
1688 }
1689 
1700 static void
1701 ril_try_unblock_agent (struct GAS_RIL_Handle *solver, struct
1702  RIL_Peer_Agent *agent, int silent)
1703 {
1704  struct RIL_Address_Wrapped *addr_wrap;
1705  struct RIL_Scope *net;
1706  unsigned long long start_in;
1707  unsigned long long start_out;
1708 
1709  for (addr_wrap = agent->addresses_head; NULL != addr_wrap; addr_wrap =
1710  addr_wrap->next)
1711  {
1712  net = addr_wrap->address_naked->solver_information;
1713  if (ril_network_is_not_full (solver, net->type))
1714  {
1715  if (NULL == agent->address_inuse)
1716  {
1717  start_in = net->bw_in_available < net->bw_in_utilized ?
1718  (net->bw_in_available - net->bw_in_utilized) / 2 :
1719  RIL_MIN_BW;
1720  start_out = net->bw_out_available < net->bw_out_utilized ?
1721  (net->bw_out_available - net->bw_out_utilized) / 2 :
1722  RIL_MIN_BW;
1723  envi_set_active_suggestion (solver, agent, addr_wrap->address_naked,
1724  start_in, start_out, silent);
1725  }
1726  return;
1727  }
1728  }
1729  agent->address_inuse = NULL;
1730 }
1731 
1738 static void
1740 {
1741  struct GNUNET_TIME_Absolute time_now;
1742  struct GNUNET_TIME_Relative time_delta;
1743  double tau;
1744 
1745  // MDP case only for debugging purposes
1746  if (solver->simulate)
1747  {
1748  solver->global_discount_variable = solver->parameters.gamma;
1749  solver->global_discount_integrated = 1;
1750  return;
1751  }
1752 
1753  // semi-MDP case
1754 
1755  // calculate tau, i.e. how many real valued time units have passed, one time unit is one minimum time step
1756  time_now = GNUNET_TIME_absolute_get ();
1758  time_now);
1759  solver->step_time_last = time_now;
1760  tau = (double) time_delta.rel_value_us
1761  / (double) solver->parameters.step_time_min.rel_value_us;
1762 
1763  // calculate reward discounts (once per step for all agents)
1764  solver->global_discount_variable = pow (M_E, ((-1.0)
1765  * ((double) solver->parameters.
1766  beta) * tau));
1767  solver->global_discount_integrated = (1.0 - solver->global_discount_variable)
1768  / (double) solver->parameters.beta;
1769 }
1770 
1778 static int
1780  RIL_Scope *scope)
1781 {
1782  int c = 0;
1783  struct RIL_Peer_Agent *cur_agent;
1784 
1785  for (cur_agent = solver->agents_head; NULL != cur_agent; cur_agent =
1786  cur_agent->next)
1787  {
1788  if (cur_agent->is_active && cur_agent->address_inuse &&
1789  (cur_agent->address_inuse->solver_information == scope))
1790  {
1791  c++;
1792  }
1793  }
1794  return c;
1795 }
1796 
1807 static unsigned long long
1809  type, int direction_in)
1810 {
1811  struct RIL_Peer_Agent *cur;
1812  struct RIL_Scope *net;
1813  unsigned long long sum = 0;
1814 
1815  for (cur = solver->agents_head; NULL != cur; cur = cur->next)
1816  {
1817  if (cur->is_active && cur->address_inuse)
1818  {
1819  net = cur->address_inuse->solver_information;
1820  if (net->type == type)
1821  {
1822  if (direction_in)
1823  sum += cur->bw_in;
1824  else
1825  sum += cur->bw_out;
1826  }
1827  }
1828  }
1829 
1830  return sum;
1831 }
1832 
1843 static unsigned long long
1845  type, int direction_in)
1846 {
1847  struct RIL_Peer_Agent *cur;
1848  struct RIL_Scope *net;
1849  unsigned long long sum = 0;
1850 
1851  for (cur = solver->agents_head; NULL != cur; cur = cur->next)
1852  {
1853  if (cur->is_active && cur->address_inuse)
1854  {
1855  net = cur->address_inuse->solver_information;
1856  if (net->type == type)
1857  {
1858  if (direction_in)
1859  sum += cur->address_inuse->norm_utilization_in.norm;
1860  else
1862  }
1863  }
1864  }
1865 
1866  return sum;
1867 }
1868 
1875 static void
1877 {
1878  int c;
1879  struct RIL_Scope *net;
1880 
1881  for (c = 0; c < solver->networks_count; c++)
1882  {
1883  net = &solver->network_entries[c];
1884  net->bw_in_assigned = ril_network_get_assigned (solver, net->type,
1885  GNUNET_YES);
1886  net->bw_in_utilized = ril_network_get_utilized (solver, net->type,
1887  GNUNET_YES);
1888  net->bw_out_assigned = ril_network_get_assigned (solver, net->type,
1889  GNUNET_NO);
1890  net->bw_out_utilized = ril_network_get_utilized (solver, net->type,
1891  GNUNET_NO);
1893  net->social_welfare = ril_network_get_social_welfare (solver, net);
1894  }
1895 }
1896 
1904 static void
1906 {
1907  double used_ratio;
1908  double factor;
1909  double y;
1910  double offset;
1911  struct GNUNET_TIME_Relative time_next;
1912 
1913  used_ratio = ril_get_used_resource_ratio (solver);
1914 
1915  GNUNET_assert (
1918 
1919  factor = (double) GNUNET_TIME_relative_subtract (
1920  solver->parameters.step_time_max,
1921  solver->parameters.
1923  offset = (double) solver->parameters.step_time_min.rel_value_us;
1924  y = factor * pow (used_ratio, RIL_INTERVAL_EXPONENT) + offset;
1925 
1926  GNUNET_assert (y <= (double) solver->parameters.step_time_max.rel_value_us);
1927  GNUNET_assert (y >= (double) solver->parameters.step_time_min.rel_value_us);
1928 
1930  GNUNET_TIME_UNIT_MICROSECONDS, (unsigned long long) y);
1931 
1932 // LOG (GNUNET_ERROR_TYPE_INFO, "ratio: %f, factor: %f, offset: %f, y: %f\n",
1933 // used_ratio,
1934 // factor,
1935 // offset,
1936 // y);
1937 
1938  if (solver->simulate)
1939  {
1940  time_next = GNUNET_TIME_UNIT_ZERO;
1941  }
1942 
1943  if ((NULL == solver->step_next_task_id) && (GNUNET_NO == solver->done))
1944  {
1945  solver->step_next_task_id = GNUNET_SCHEDULER_add_delayed (time_next,
1946  &
1948  solver);
1949  }
1950 }
1951 
1957 static void
1958 ril_step (struct GAS_RIL_Handle *solver)
1959 {
1960  struct RIL_Peer_Agent *cur;
1961 
1962  if (GNUNET_YES == solver->bulk_lock)
1963  {
1964  solver->bulk_changes++;
1965  return;
1966  }
1967 
1969 
1970  LOG (GNUNET_ERROR_TYPE_DEBUG, " RIL step number %d\n", solver->step_count);
1971 
1972  if (0 == solver->step_count)
1973  {
1975  }
1976 
1977  ril_calculate_discount (solver);
1978  ril_networks_update_state (solver);
1979 
1980  // trigger one step per active, unblocked agent
1981  for (cur = solver->agents_head; NULL != cur; cur = cur->next)
1982  {
1983  if (cur->is_active)
1984  {
1985  if (NULL == cur->address_inuse)
1986  {
1987  ril_try_unblock_agent (solver, cur, GNUNET_NO);
1988  }
1989  if (cur->address_inuse)
1990  {
1991  agent_step (cur);
1992  }
1993  }
1994  }
1995 
1996  ril_networks_update_state (solver);
1997 
1998  solver->step_count++;
1999  ril_step_schedule_next (solver);
2000 
2002 
2004  for (cur = solver->agents_head; NULL != cur; cur = cur->next)
2005  {
2006  if (cur->suggestion_issue)
2007  {
2008  solver->env->bandwidth_changed_cb (solver->env->cls,
2009  cur->suggestion_address);
2010  cur->suggestion_issue = GNUNET_NO;
2011  }
2012  }
2014 }
2015 
2021 static void
2023 {
2024  int i;
2025  int k;
2026 
2027  for (i = 0; i < agent->n; i++)
2028  {
2029  for (k = 0; k < agent->m; k++)
2030  {
2031  agent->W[i][k] = agent->envi->parameters.alpha * (1.0 - 2.0
2032  * ((double)
2035  UINT32_MAX)
2036  / (double) UINT32_MAX));
2037  }
2038  }
2039 }
2040 
2048 static struct RIL_Peer_Agent *
2049 agent_init (void *s, const struct GNUNET_PeerIdentity *peer)
2050 {
2051  int i;
2052  struct GAS_RIL_Handle *solver = s;
2053  struct RIL_Peer_Agent *agent = GNUNET_new (struct RIL_Peer_Agent);
2054 
2055  agent->envi = solver;
2056  agent->peer = *peer;
2057  agent->step_count = 0;
2058  agent->is_active = GNUNET_NO;
2059  agent->bw_in = RIL_MIN_BW;
2060  agent->bw_out = RIL_MIN_BW;
2061  agent->suggestion_issue = GNUNET_NO;
2062  agent->n = RIL_ACTION_TYPE_NUM;
2063  agent->m = 0;
2064  agent->W = (double **) GNUNET_malloc (sizeof(double *) * agent->n);
2065  agent->E = (double **) GNUNET_malloc (sizeof(double *) * agent->n);
2066  for (i = 0; i < agent->n; i++)
2067  {
2068  agent->W[i] = (double *) GNUNET_malloc (sizeof(double) * agent->m);
2069  agent->E[i] = (double *) GNUNET_malloc (sizeof(double) * agent->m);
2070  }
2071  agent_w_init (agent);
2072  agent->eligibility_reset = GNUNET_NO;
2073  agent->a_old = RIL_ACTION_INVALID;
2074  agent->s_old = GNUNET_malloc (sizeof(double) * agent->m);
2075  agent->address_inuse = NULL;
2076  agent->objective_old = 0;
2077  agent->nop_bonus = 0;
2078 
2079  return agent;
2080 }
2081 
2088 static void
2089 agent_die (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent)
2090 {
2091  int i;
2092 
2093  for (i = 0; i < agent->n; i++)
2094  {
2095  GNUNET_free_non_null (agent->W[i]);
2096  GNUNET_free_non_null (agent->E[i]);
2097  }
2098  GNUNET_free_non_null (agent->W);
2099  GNUNET_free_non_null (agent->E);
2100  GNUNET_free_non_null (agent->s_old);
2101  GNUNET_free (agent);
2102 }
2103 
2112 static struct RIL_Peer_Agent *
2113 ril_get_agent (struct GAS_RIL_Handle *solver, const struct
2114  GNUNET_PeerIdentity *peer, int create)
2115 {
2116  struct RIL_Peer_Agent *cur;
2117 
2118  for (cur = solver->agents_head; NULL != cur; cur = cur->next)
2119  {
2120  if (0 == GNUNET_memcmp (peer, &cur->peer))
2121  {
2122  return cur;
2123  }
2124  }
2125 
2126  if (create)
2127  {
2128  cur = agent_init (solver, peer);
2130  cur);
2131  return cur;
2132  }
2133  return NULL;
2134 }
2135 
2144 static int
2146  network)
2147 {
2148  struct RIL_Scope *net;
2149 
2150  net = ril_get_network (solver, network);
2151  return net->bw_out_available >= RIL_MIN_BW;
2152 }
2153 
2165 static void
2167  size_t element_size,
2168  unsigned int hole_start,
2169  unsigned int hole_length,
2170  unsigned int old_length)
2171 {
2172  char *tmpptr;
2173  char *oldptr = (char *) *old;
2174  size_t size;
2175  unsigned int bytes_before;
2176  unsigned int bytes_hole;
2177  unsigned int bytes_after;
2178 
2179  GNUNET_assert (old_length >= hole_length);
2180  GNUNET_assert (old_length >= (hole_start + hole_length));
2181 
2182  size = element_size * (old_length - hole_length);
2183 
2184  bytes_before = element_size * hole_start;
2185  bytes_hole = element_size * hole_length;
2186  bytes_after = element_size * (old_length - hole_start - hole_length);
2187 
2188  if (0 == size)
2189  {
2190  tmpptr = NULL;
2191  }
2192  else
2193  {
2194  tmpptr = GNUNET_malloc (size);
2195  GNUNET_memcpy (tmpptr, oldptr, bytes_before);
2196  GNUNET_memcpy (tmpptr + bytes_before, oldptr + (bytes_before + bytes_hole),
2197  bytes_after);
2198  }
2199  if (NULL != *old)
2200  {
2201  GNUNET_free (*old);
2202  }
2203  *old = (void *) tmpptr;
2204 }
2205 
2206 /*
2207  * Solver API functions
2208  * ---------------------------
2209  */
2210 
2219 static void
2221  const struct GNUNET_PeerIdentity *peer,
2222  enum GNUNET_ATS_PreferenceKind kind,
2223  double pref_rel)
2224 {
2226  "API_address_change_preference() Preference '%s' for peer '%s' changed to %.2f \n",
2227  GNUNET_ATS_print_preference_type (kind), GNUNET_i2s (peer), pref_rel);
2228 
2229  struct GAS_RIL_Handle *s = solver;
2230 
2233  ril_step (s);
2234 }
2235 
2236 
2246 static void
2247 GAS_ril_address_add (void *solver,
2248  struct ATS_Address *address,
2249  uint32_t network)
2250 {
2251  struct GAS_RIL_Handle *s = solver;
2252  struct RIL_Peer_Agent *agent;
2253  struct RIL_Address_Wrapped *address_wrapped;
2254  struct RIL_Scope *net;
2255  unsigned int m_new;
2256  unsigned int m_old;
2257  unsigned int n_new;
2258  unsigned int n_old;
2259  int i;
2260  unsigned int zero;
2261 
2263  "API_address_add()\n");
2264 
2265  net = ril_get_network (s, network);
2266  address->solver_information = net;
2267 
2268  if (! ril_network_is_active (s, network))
2269  {
2271  "API_address_add() Did not add %s address %s for peer '%s', network does not have enough bandwidth\n",
2272  address->plugin, address->addr, GNUNET_i2s (&address->peer));
2273  return;
2274  }
2275 
2278 
2279  agent = ril_get_agent (s, &address->peer, GNUNET_YES);
2280 
2281  // add address
2282  address_wrapped = GNUNET_new (struct RIL_Address_Wrapped);
2283  address_wrapped->address_naked = address;
2285  agent->addresses_tail, address_wrapped);
2286 
2287  // increase size of W
2288  m_new = agent->m + ((s->parameters.rbf_divisor + 1)
2289  * (s->parameters.rbf_divisor + 1));
2290  m_old = agent->m;
2291  n_new = agent->n + 1;
2292  n_old = agent->n;
2293 
2294  GNUNET_array_grow (agent->W, agent->n, n_new);
2295  agent->n = n_old;
2296  GNUNET_array_grow (agent->E, agent->n, n_new);
2297  for (i = 0; i < n_new; i++)
2298  {
2299  if (i < n_old)
2300  {
2301  agent->m = m_old;
2302  GNUNET_array_grow (agent->W[i], agent->m, m_new);
2303  agent->m = m_old;
2304  GNUNET_array_grow (agent->E[i], agent->m, m_new);
2305  }
2306  else
2307  {
2308  zero = 0;
2309  GNUNET_array_grow (agent->W[i], zero, m_new);
2310  zero = 0;
2311  GNUNET_array_grow (agent->E[i], zero, m_new);
2312  }
2313  }
2314 
2315  // increase size of old state vector
2316  agent->m = m_old;
2317  GNUNET_array_grow (agent->s_old, agent->m, m_new);
2318 
2319  ril_try_unblock_agent (s, agent, GNUNET_NO);
2320 
2321  ril_step (s);
2322 
2324  "API_address_add() Added %s %s address %s for peer '%s'\n",
2325  address->active ? "active" : "inactive", address->plugin, address->addr,
2326  GNUNET_i2s (&address->peer));
2327 }
2328 
2337 static void
2339  struct ATS_Address *address)
2340 {
2341  struct GAS_RIL_Handle *s = solver;
2342  struct RIL_Peer_Agent *agent;
2343  struct RIL_Address_Wrapped *address_wrapped;
2344  int address_index;
2345  unsigned int m_new;
2346  unsigned int n_new;
2347  int i;
2348  struct RIL_Scope *net;
2349 
2351  "API_address_delete() Delete %s %s address %s for peer '%s'\n",
2352  address->active ? "active" : "inactive",
2353  address->plugin,
2354  address->addr,
2355  GNUNET_i2s (&address->peer));
2356 
2357  agent = ril_get_agent (s, &address->peer, GNUNET_NO);
2358  if (NULL == agent)
2359  {
2360  net = address->solver_information;
2363  "No agent allocated for peer yet, since address was in inactive network\n");
2364  return;
2365  }
2366 
2369 
2370  address_index = agent_address_get_index (agent, address);
2371  address_wrapped = agent_address_get_wrapped (agent, address);
2372 
2373  if (NULL == address_wrapped)
2374  {
2375  net = address->solver_information;
2377  "Address not considered by agent, address was in inactive network\n");
2378  return;
2379  }
2381  agent->addresses_tail,
2382  address_wrapped);
2383  GNUNET_free (address_wrapped);
2384 
2385  // decrease W
2386  m_new = agent->m - ((s->parameters.rbf_divisor + 1)
2387  * (s->parameters.rbf_divisor + 1));
2388  n_new = agent->n - 1;
2389 
2390  for (i = 0; i < agent->n; i++)
2391  {
2392  ril_cut_from_vector ((void **) &agent->W[i], sizeof(double),
2393  address_index * ((s->parameters.rbf_divisor + 1)
2394  * (s->parameters.rbf_divisor + 1)),
2395  ((s->parameters.rbf_divisor + 1)
2396  * (s->parameters.rbf_divisor + 1)), agent->m);
2397  ril_cut_from_vector ((void **) &agent->E[i], sizeof(double),
2398  address_index * ((s->parameters.rbf_divisor + 1)
2399  * (s->parameters.rbf_divisor + 1)),
2400  ((s->parameters.rbf_divisor + 1)
2401  * (s->parameters.rbf_divisor + 1)), agent->m);
2402  }
2403  GNUNET_free_non_null (agent->W[RIL_ACTION_TYPE_NUM + address_index]);
2404  GNUNET_free_non_null (agent->E[RIL_ACTION_TYPE_NUM + address_index]);
2405  ril_cut_from_vector ((void **) &agent->W, sizeof(double *),
2406  RIL_ACTION_TYPE_NUM + address_index,
2407  1, agent->n);
2408  ril_cut_from_vector ((void **) &agent->E, sizeof(double *),
2409  RIL_ACTION_TYPE_NUM + address_index,
2410  1, agent->n);
2411  // correct last action
2412  if (agent->a_old > (RIL_ACTION_TYPE_NUM + address_index))
2413  {
2414  agent->a_old -= 1;
2415  }
2416  else if (agent->a_old == (RIL_ACTION_TYPE_NUM + address_index))
2417  {
2418  agent->a_old = RIL_ACTION_INVALID;
2419  }
2420  // decrease old state vector
2421  ril_cut_from_vector ((void **) &agent->s_old, sizeof(double),
2422  address_index * ((s->parameters.rbf_divisor + 1)
2423  * (s->parameters.rbf_divisor + 1)),
2424  ((s->parameters.rbf_divisor + 1)
2425  * (s->parameters.rbf_divisor + 1)), agent->m);
2426  agent->m = m_new;
2427  agent->n = n_new;
2428 
2429  if (agent->address_inuse == address)
2430  {
2431  if (NULL != agent->addresses_head) // if peer has an address left, use it
2432  {
2434  "Active address died, suggesting alternative!\n");
2436  agent,
2437  agent->addresses_head->address_naked,
2438  agent->bw_in,
2439  agent->bw_out,
2440  GNUNET_YES);
2441  }
2442  else
2443  {
2445  "Active address died, suggesting disconnect!\n");
2446  envi_set_active_suggestion (s, agent, NULL, 0, 0, GNUNET_NO);
2447  }
2448  }
2449  ril_step (solver);
2450  if (agent->suggestion_address == address)
2451  {
2452  agent->suggestion_issue = GNUNET_NO;
2453  agent->suggestion_address = NULL;
2454  }
2455  GNUNET_assert (agent->address_inuse != address);
2456 }
2457 
2458 
2465 static void
2467  struct ATS_Address *address)
2468 {
2469  struct GAS_RIL_Handle *s = solver;
2470 
2472  "Properties for peer '%s' address changed\n",
2473  GNUNET_i2s (&address->peer));
2476  ril_step (s);
2477 }
2478 
2479 
2490 static void
2492  struct GNUNET_SERVICE_Client *application,
2493  const struct GNUNET_PeerIdentity *peer,
2494  const struct GNUNET_TIME_Relative scope,
2495  enum GNUNET_ATS_PreferenceKind kind,
2496  double score)
2497 {
2499  "API_address_preference_feedback() Peer '%s' got a feedback of %+.3f from application %s for "
2500  "preference %s for %d seconds\n",
2501  GNUNET_i2s (peer),
2502  "UNKNOWN",
2504  scope.rel_value_us / 1000000);
2505 }
2506 
2507 
2513 static void
2514 GAS_ril_bulk_start (void *solver)
2515 {
2516  struct GAS_RIL_Handle *s = solver;
2517 
2519  "API_bulk_start() lock: %d\n", s->bulk_lock + 1);
2520 
2521  s->bulk_lock++;
2522 }
2523 
2524 
2530 static void
2531 GAS_ril_bulk_stop (void *solver)
2532 {
2533  struct GAS_RIL_Handle *s = solver;
2534 
2536  "API_bulk_stop() lock: %d\n",
2537  s->bulk_lock - 1);
2538 
2539  if (s->bulk_lock < 1)
2540  {
2541  GNUNET_break (0);
2542  return;
2543  }
2544  s->bulk_lock--;
2545 
2546  if (0 < s->bulk_changes)
2547  {
2548  ril_step (solver);
2549  s->bulk_changes = 0;
2550  }
2551 }
2552 
2553 
2564 static void
2566  const struct GNUNET_PeerIdentity *peer)
2567 {
2568  struct GAS_RIL_Handle *s = solver;
2569  struct RIL_Peer_Agent *agent;
2570 
2571  LOG (GNUNET_ERROR_TYPE_DEBUG, "API_get_preferred_address()\n");
2572 
2573  agent = ril_get_agent (s, peer, GNUNET_YES);
2574 
2575  agent->is_active = GNUNET_YES;
2576  envi_set_active_suggestion (solver, agent, agent->address_inuse, agent->bw_in,
2577  agent->bw_out, GNUNET_YES);
2578 
2579  ril_try_unblock_agent (solver, agent, GNUNET_YES);
2580 
2581  if (agent->address_inuse)
2582  {
2584  "API_get_preferred_address() Activated agent for peer '%s' with %s address %s\n",
2585  GNUNET_i2s (peer), agent->address_inuse->plugin,
2586  agent->address_inuse->addr);
2587  }
2588  else
2589  {
2591  "API_get_preferred_address() Activated agent for peer '%s', but no address available\n",
2592  GNUNET_i2s (peer));
2595  }
2596  if (NULL != agent->address_inuse)
2597  s->env->bandwidth_changed_cb (s->env->cls,
2598  agent->address_inuse);
2599 }
2600 
2601 
2611 static void
2613  const struct GNUNET_PeerIdentity *peer)
2614 {
2615  struct GAS_RIL_Handle *s = solver;
2616  struct RIL_Peer_Agent *agent;
2617 
2619  "API_stop_get_preferred_address()");
2620 
2621  agent = ril_get_agent (s, peer, GNUNET_NO);
2622 
2623  if (NULL == agent)
2624  {
2625  GNUNET_break (0);
2626  return;
2627  }
2628  if (GNUNET_NO == agent->is_active)
2629  {
2630  GNUNET_break (0);
2631  return;
2632  }
2633 
2636 
2637  agent->is_active = GNUNET_NO;
2638 
2639  envi_set_active_suggestion (s, agent, agent->address_inuse, agent->bw_in,
2640  agent->bw_out,
2641  GNUNET_YES);
2642 
2643  ril_step (s);
2644 
2646  "API_stop_get_preferred_address() Paused agent for peer '%s'\n",
2647  GNUNET_i2s (peer));
2648 }
2649 
2650 
2656 void *
2658 {
2659  static struct GNUNET_ATS_SolverFunctions sf;
2661  struct GAS_RIL_Handle *solver = GNUNET_new (struct GAS_RIL_Handle);
2662  struct RIL_Scope *cur;
2663  int c;
2664  char *string;
2665  float f_tmp;
2666 
2668  "API_init() Initializing RIL solver\n");
2669 
2670  GNUNET_assert (NULL != env);
2671  GNUNET_assert (NULL != env->cfg);
2672  GNUNET_assert (NULL != env->stats);
2673  GNUNET_assert (NULL != env->bandwidth_changed_cb);
2674  GNUNET_assert (NULL != env->get_preferences);
2675 
2677  "RIL_RBF_DIVISOR",
2678  &solver->parameters.
2679  rbf_divisor))
2680  {
2682  }
2683 
2684  if (GNUNET_OK
2685  != GNUNET_CONFIGURATION_get_value_time (env->cfg, "ats",
2686  "RIL_STEP_TIME_MIN",
2687  &solver->parameters.step_time_min))
2688  {
2690  }
2691 
2692  if (GNUNET_OK
2693  != GNUNET_CONFIGURATION_get_value_time (env->cfg, "ats",
2694  "RIL_STEP_TIME_MAX",
2695  &solver->parameters.step_time_max))
2696  {
2698  }
2699 
2701  "RIL_ALGORITHM",
2702  &string))
2703  {
2704  GNUNET_STRINGS_utf8_toupper (string, string);
2705  if (0 == strcmp (string, "SARSA"))
2706  {
2708  }
2709  if (0 == strcmp (string, "Q-LEARNING"))
2710  {
2711  solver->parameters.algorithm = RIL_ALGO_Q;
2712  }
2713 
2714  GNUNET_free (string);
2715  }
2716  else
2717  {
2719  }
2720 
2722  "RIL_SELECT",
2723  &string))
2724  {
2725  solver->parameters.select = ! strcmp (string, "EGREEDY") ?
2727  GNUNET_free (string);
2728  }
2729  else
2730  {
2732  }
2733 
2734 
2737  "RIL_DISCOUNT_BETA",
2738  &f_tmp))
2739  {
2740  if (f_tmp < 0.0)
2741  {
2742  LOG (GNUNET_ERROR_TYPE_ERROR, _ ("Invalid %s configuration %f \n"),
2743  "RIL_DISCOUNT_BETA", f_tmp);
2744  }
2745  else
2746  {
2747  solver->parameters.beta = f_tmp;
2748  LOG (GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n",
2749  "RIL_DISCOUNT_BETA", f_tmp);
2750  }
2751  }
2752 
2755  "RIL_DISCOUNT_GAMMA",
2756  &f_tmp))
2757  {
2758  if ((f_tmp < 0.0) || (f_tmp > 1.0))
2759  {
2760  LOG (GNUNET_ERROR_TYPE_ERROR, _ ("Invalid %s configuration %f \n"),
2761  "RIL_DISCOUNT_GAMMA", f_tmp);
2762  }
2763  else
2764  {
2765  solver->parameters.gamma = f_tmp;
2766  LOG (GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n",
2767  "RIL_DISCOUNT_GAMMA", f_tmp);
2768  }
2769  }
2770 
2773  "RIL_GRADIENT_STEP_SIZE",
2774  &f_tmp))
2775  {
2776  if ((f_tmp < 0.0) || (f_tmp > 1.0))
2777  {
2778  LOG (GNUNET_ERROR_TYPE_ERROR, _ ("Invalid %s configuration %f \n"),
2779  "RIL_GRADIENT_STEP_SIZE", f_tmp);
2780  }
2781  else
2782  {
2783  solver->parameters.alpha = f_tmp;
2784  LOG (GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n",
2785  "RIL_GRADIENT_STEP_SIZE", f_tmp);
2786  }
2787  }
2788 
2791  "RIL_TRACE_DECAY",
2792  &f_tmp))
2793  {
2794  if ((f_tmp < 0.0) || (f_tmp > 1.0))
2795  {
2796  LOG (GNUNET_ERROR_TYPE_ERROR, _ ("Invalid %s configuration %f \n"),
2797  "RIL_TRACE_DECAY", f_tmp);
2798  }
2799  else
2800  {
2801  solver->parameters.lambda = f_tmp;
2802  LOG (GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n",
2803  "RIL_TRACE_DECAY", f_tmp);
2804  }
2805  }
2806 
2809  "RIL_EXPLORE_RATIO",
2810  &f_tmp))
2811  {
2812  if ((f_tmp < 0.0) || (f_tmp > 1.0))
2813  {
2814  LOG (GNUNET_ERROR_TYPE_ERROR, _ ("Invalid %s configuration %f \n"),
2815  "RIL_EXPLORE_RATIO", f_tmp);
2816  }
2817  else
2818  {
2819  solver->parameters.epsilon_init = f_tmp;
2820  LOG (GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n",
2821  "RIL_EXPLORE_RATIO", f_tmp);
2822  }
2823  }
2824 
2827  "RIL_EXPLORE_DECAY",
2828  &f_tmp))
2829  {
2830  if ((f_tmp < 0.0) || (f_tmp > 1.0))
2831  {
2832  LOG (GNUNET_ERROR_TYPE_ERROR, _ ("Invalid %s configuration %f \n"),
2833  "RIL_EXPLORE_DECAY", f_tmp);
2834  }
2835  else
2836  {
2837  solver->parameters.epsilon_decay = f_tmp;
2838  LOG (GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n",
2839  "RIL_EXPLORE_DECAY", f_tmp);
2840  }
2841  }
2842 
2845  "RIL_TEMPERATURE",
2846  &f_tmp))
2847  {
2848  if (f_tmp <= 0.0)
2849  {
2850  LOG (GNUNET_ERROR_TYPE_ERROR, _ ("Invalid %s configuration %f \n"),
2851  "RIL_TEMPERATURE", f_tmp);
2852  }
2853  else
2854  {
2855  solver->parameters.temperature_init = f_tmp;
2856  LOG (GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n",
2857  "RIL_TEMPERATURE", f_tmp);
2858  }
2859  }
2860 
2863  "RIL_TEMPERATURE_DECAY",
2864  &f_tmp))
2865  {
2866  if ((f_tmp <= 0.0) ||(solver->parameters.temperature_decay > 1) )
2867  {
2868  LOG (GNUNET_ERROR_TYPE_ERROR, _ ("Invalid %s configuration %f \n"),
2869  "RIL_TEMPERATURE_DECAY", f_tmp);
2870  }
2871  else
2872  {
2873  solver->parameters.temperature_decay = f_tmp;
2874  LOG (GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n",
2875  "RIL_TEMPERATURE_DECAY", f_tmp);
2876  }
2877  }
2878 
2880  "RIL_SIMULATE",
2881  &solver->simulate))
2882  {
2883  solver->simulate = GNUNET_NO;
2884  }
2885 
2887  "RIL_REPLACE_TRACES"))
2888  {
2890  }
2891  else
2892  {
2894  }
2895 
2897  "RIL_SOCIAL_WELFARE",
2898  &string))
2899  {
2900  solver->parameters.social_welfare = ! strcmp (string, "NASH") ?
2903  GNUNET_free (string);
2904  }
2905  else
2906  {
2908  }
2909 
2910  solver->env = env;
2911  sf.cls = solver;
2912  sf.s_add = &GAS_ril_address_add;
2921 
2922  solver->networks_count = env->network_count;
2923  solver->network_entries = GNUNET_malloc (env->network_count * sizeof(struct
2924  RIL_Scope));
2925  solver->step_count = 0;
2926  solver->done = GNUNET_NO;
2927 
2928  for (c = 0; c < env->network_count; c++)
2929  {
2930  cur = &solver->network_entries[c];
2931  cur->type = c;
2932  cur->bw_in_available = env->in_quota[c];
2933  cur->bw_out_available = env->out_quota[c];
2935  "init() Quotas for %s network: IN %llu - OUT %llu\n",
2936  GNUNET_NT_to_string (cur->type),
2937  cur->bw_in_available / 1024,
2938  cur->bw_out_available / 1024);
2939  }
2940 
2941  LOG (GNUNET_ERROR_TYPE_DEBUG, "init() Parameters:\n");
2943  "init() Algorithm = %s, alpha = %f, beta = %f, lambda = %f\n",
2944  solver->parameters.algorithm ? "Q" : "SARSA",
2945  solver->parameters.alpha,
2946  solver->parameters.beta,
2947  solver->parameters.lambda);
2949  "init() exploration_ratio = %f, temperature = %f, ActionSelection = %s\n",
2950  solver->parameters.epsilon,
2951  solver->parameters.temperature,
2952  solver->parameters.select ? "EGREEDY" : "SOFTMAX");
2953  LOG (GNUNET_ERROR_TYPE_DEBUG, "init() RBF_DIVISOR = %llu\n",
2954  solver->parameters.rbf_divisor);
2955 
2956  return &sf;
2957 }
2958 
2959 
2965 void *
2967 {
2968  struct GNUNET_ATS_SolverFunctions *sf = cls;
2969  struct GAS_RIL_Handle *s = sf->cls;
2970  struct RIL_Peer_Agent *cur_agent;
2971  struct RIL_Peer_Agent *next_agent;
2972 
2973  LOG (GNUNET_ERROR_TYPE_DEBUG, "API_done() Shutting down RIL solver\n");
2974 
2975  s->done = GNUNET_YES;
2976 
2977  cur_agent = s->agents_head;
2978  while (NULL != cur_agent)
2979  {
2980  next_agent = cur_agent->next;
2982  agent_die (s, cur_agent);
2983  cur_agent = next_agent;
2984  }
2985 
2986  if (NULL != s->step_next_task_id)
2987  {
2989  }
2991  GNUNET_free (s);
2992 
2993  return NULL;
2994 }
2995 
2996 
2997 /* end of plugin_ats_ril.c */
struct RIL_Address_Wrapped * prev
Previous in DLL.
#define GNUNET_CONTAINER_DLL_remove(head, tail, element)
Remove an element from a DLL.
struct GNUNET_SCHEDULER_Task * step_next_task_id
Task identifier of the next time-step to be executed.
static const struct GNUNET_CRYPTO_EcdsaPrivateKey zero
Public key of all zeros.
GAS_solver_stop_get_preferred_address s_get_stop
Tell solver stop notifying ATS about changes for this peers.
static struct RIL_Scope * ril_get_network(struct GAS_RIL_Handle *s, uint32_t type)
Lookup network struct by type.
static void GAS_ril_address_change_preference(void *solver, const struct GNUNET_PeerIdentity *peer, enum GNUNET_ATS_PreferenceKind kind, double pref_rel)
Change relative preference for quality in solver.
static void GAS_ril_address_delete(void *solver, struct ATS_Address *address)
Delete an address in the solver.
After the problem was finished, start notifications about changes to addresses.
#define RIL_MAX_BW
struct RIL_Peer_Agent * agents_head
List of active peer-agents.
uint32_t bw_in
Inbound bandwidth assigned by the agent.
int is_active
Whether the agent is active or not.
struct GNUNET_ATS_PluginEnvironment * env
The solver-plugin environment of the solver-plugin API.
Global learning parameters.
int GNUNET_CONFIGURATION_get_value_time(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, struct GNUNET_TIME_Relative *time)
Get a configuration value that should be a relative time.
GAS_solver_address_add s_add
Add a new address for a peer to the solver.
double global_discount_integrated
Integrated variable discount factor, dependent on time between steps.
int GNUNET_CONFIGURATION_get_value_number(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, unsigned long long *number)
Get a configuration value that should be a number.
static void envi_action_bw_halven(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int direction_in)
Cuts the bandwidth for the active address in half.
uint64_t rel_value_us
The actual value.
const char * GNUNET_NT_to_string(enum GNUNET_NetworkType net)
Convert a enum GNUNET_NetworkType to a string.
Definition: nt.c:44
static struct GNUNET_TIME_Relative delta
Definition: speedup.c:35
#define RIL_DEFAULT_SELECT
static unsigned int element_size
double lambda
Trace-decay factor for eligibility traces.
#define RIL_UTILITY_DELAY_MAX
static int agent_select_softmax(struct RIL_Peer_Agent *agent, double *state)
Selects the next action with a probability corresponding to its value.
struct GNUNET_STATISTICS_Handle * stats
Statistics handle to be used by the solver.
static void agent_die(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent)
Deallocate agent.
#define RIL_DEFAULT_DISCOUNT_GAMMA
#define RIL_DEFAULT_EXPLORE_DECAY
#define RIL_DEFAULT_RBF_DIVISOR
int bulk_changes
Number of changes during a lock.
static void GAS_ril_address_add(void *solver, struct ATS_Address *address, uint32_t network)
Add a new address for a peer to the solver.
int active
Is this the active address for this peer?
GAS_get_preferences get_preferences
ATS addresses function to obtain preference values.
unsigned int m
Number of rows of W / Number of state-vector features.
static void GAS_ril_bulk_stop(void *solver)
Bulk operation done.
struct GNUNET_PeerIdentity peer
Peer ID this address is for.
GAS_Solver_Status
Status of a GAS_Solver_Operation operation.
uint32_t GNUNET_CRYPTO_random_u32(enum GNUNET_CRYPTO_Quality mode, uint32_t i)
Produce a random value.
static double agent_q(struct RIL_Peer_Agent *agent, const double *state, int action)
Estimate the current action-value for state s and action a.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
static void ril_inform(struct GAS_RIL_Handle *solver, enum GAS_Solver_Operation op, enum GAS_Solver_Status stat)
Informs the environment about the status of the solver.
static void GAS_ril_stop_get_preferred_address(void *solver, const struct GNUNET_PeerIdentity *peer)
Tell solver stop notifying ATS about changes for this peers.
static void envi_action_bw_dec(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int direction_in)
Decreases the bandwidth by 5 times the minimum bandwidth for the active address.
No more specific information.
A handle for the reinforcement learning solver.
const void * addr
Address (in plugin-specific binary format).
static int agent_get_action_max(struct RIL_Peer_Agent *agent, double *state)
Gets the action, with the maximal estimated Q-value (i.e.
struct RIL_Address_Wrapped * addresses_head
Head of addresses DLL.
int eligibility_reset
Whether to reset the eligibility traces to 0 after a Q-exploration step.
double gamma
Learning discount factor in the TD-update for MDPs.
struct GNUNET_TIME_Relative step_time_max
Maximum interval time between steps in milliseconds.
#define RIL_DEFAULT_GRADIENT_STEP_SIZE
GAS_solver_address_feedback_preference s_feedback
Give feedback about the current assignment.
#define GNUNET_memcpy(dst, src, n)
Call memcpy() but check for n being 0 first.
int a_old
Last chosen action.
int GNUNET_CONFIGURATION_get_value_float(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, float *number)
Get a configuration value that should be a floating point number.
static void ril_cut_from_vector(void **old, size_t element_size, unsigned int hole_start, unsigned int hole_length, unsigned int old_length)
Cuts a slice out of a vector of elements.
GAS_solver_address_property_changed s_address_update_property
Update the properties of an address in the solver.
double temperature_decay
Decay factor of the temperature value.
#define RIL_DEFAULT_WELFARE
#define GNUNET_NO
Definition: gnunet_common.h:78
struct RIL_Scope * network_entries
Array of networks with global assignment state.
#define GNUNET_OK
Named constants for return values.
Definition: gnunet_common.h:75
#define GNUNET_free_non_null(ptr)
Free the memory pointed to by ptr if ptr is not NULL.
char * plugin
Plugin name.
int suggestion_issue
Flag whether a suggestion has to be issued.
#define GNUNET_new(type)
Allocate a struct or union of the given type.
struct RIL_Peer_Agent * agents_tail
#define RIL_INTERVAL_EXPONENT
static void ril_try_unblock_agent(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int silent)
Unblocks an agent for which a connection request is there, that could not be satisfied.
void * libgnunet_plugin_ats_ril_init(void *cls)
Entry point for the plugin.
double epsilon
Ratio, with what probability an agent should explore in the e-greed policy.
static int ril_network_is_active(struct GAS_RIL_Handle *solver, enum GNUNET_NetworkType network)
Determine whether at least the minimum bandwidth is set for the network.
static int ril_network_is_not_full(struct GAS_RIL_Handle *solver, enum GNUNET_NetworkType network)
Determines whether more connections are allocated in a network scope, than they would theoretically f...
static void envi_set_active_suggestion(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, struct ATS_Address *new_address, unsigned long long new_bw_in, unsigned long long new_bw_out, int silent)
Changes the active assignment suggestion of the handler and invokes the bw_changed callback to notify...
static void GAS_ril_address_property_changed(void *solver, struct ATS_Address *address)
Update the properties of an address in the solver.
struct ATS_Address * address_naked
The address.
RIL_E_Modification
static void envi_action_address_switch(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, unsigned int address_index)
Switches to the address given by its index.
GNUNET_NetworkType
Types of networks (with separate quotas) we support.
Definition: gnunet_nt_lib.h:35
static double * envi_get_state(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent)
Allocates a state vector and fills it with the features present.
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur...
unsigned long long step_count
Number of performed steps.
RIL_Welfare
double temperature
Softmax action-selection temperature.
#define _(String)
GNU gettext support macro.
Definition: platform.h:181
static void ril_step_scheduler_task(void *cls)
Task for the scheduler, which performs one step and lets the solver know that no further step is sche...
GAS_Solver_Operation
Operation codes for solver information callback.
Handle to a client that is connected to a service.
Definition: service.c:250
struct GNUNET_TIME_Absolute step_time_last
Timestamp for the last time-step.
uint32_t assigned_bw_in
Inbound bandwidth assigned by solver.
static int agent_get_action_random(struct RIL_Peer_Agent *agent)
Chooses a random action from the set of possible ones.
struct GNUNET_PeerIdentity peer
Peer ID.
static struct GNUNET_ARM_MonitorHandle * m
Monitor connection with ARM.
Definition: gnunet-arm.c:104
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_delayed(struct GNUNET_TIME_Relative delay, GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run with a specified delay.
Definition: scheduler.c:1253
#define RIL_DEFAULT_TRACE_DECAY
unsigned long long simulate
Simulate steps, i.e.
A solution iteration has been started.
#define GNUNET_array_grow(arr, size, tsize)
Grow a well-typed (!) array.
void GNUNET_STRINGS_utf8_toupper(const char *input, char *output)
Convert the utf-8 input string to upper case.
Definition: strings.c:578
enum State state
current state of profiling
static void GAS_ril_get_preferred_address(void *solver, const struct GNUNET_PeerIdentity *peer)
Tell solver to notify ATS if the address to use changes for a specific peer using the bandwidth chang...
#define RIL_DEFAULT_ALGORITHM
static struct GNUNET_OS_Process * p
Helper process we started.
Definition: gnunet-qr.c:59
enum RIL_Select select
Action selection strategy;.
static void envi_do_action(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int action)
Puts the action into effect by calling the according function.
static void ril_step_schedule_next(struct GAS_RIL_Handle *solver)
Schedules the next global step in an adaptive way.
unsigned long long bw_out_utilized
Bandwidth outbound actually utilized in the network.
static struct RIL_Peer_Agent * agent_init(void *s, const struct GNUNET_PeerIdentity *peer)
Initialize an agent without addresses and its knowledge base.
static int silent
–silent option
Definition: gnunet-qr.c:49
enum GNUNET_NetworkType type
ATS network type.
#define GNUNET_MAX(a, b)
Definition: gnunet_common.h:82
static enum GNUNET_NetworkType scope
Which network scope do we belong to?
enum RIL_Welfare social_welfare
Which measure of social welfare should be used.
static void agent_w_init(struct RIL_Peer_Agent *agent)
Initializes the matrix W of parameter vectors theta with small random numbers.
double beta
Learning discount variable in the TD-update for semi-MDPs.
static unsigned long long ril_network_get_utilized(struct GAS_RIL_Handle *solver, enum GNUNET_NetworkType type, int direction_in)
Calculates how much bandwidth is actually utilized in sum in a network scope, either in the inbound o...
static unsigned long long ril_get_max_bw(struct RIL_Scope *net)
Calculates the maximum bandwidth an agent can assign in a network scope.
static struct GNUNET_PEERINFO_NotifyContext * notify
Handle to the peerinfo notify service (NULL until we&#39;ve connected to it).
#define GNUNET_MIN(a, b)
Definition: gnunet_common.h:80
GAS_solver_address_delete s_del
Delete an address in the solver.
static int agent_address_get_index(struct RIL_Peer_Agent *agent, struct ATS_Address *address)
Get the index of the address in the agent&#39;s list.
#define RIL_DEFAULT_TEMPERATURE_DECAY
static int result
Global testing status.
enum RIL_Algorithm algorithm
The TD-algorithm to use.
#define RIL_DEFAULT_STEP_TIME_MAX
double ** W
Experience matrix W.
static int agent_select_egreedy(struct RIL_Peer_Agent *agent, double *state)
Selects the next action using the e-greedy strategy.
struct GAS_NormalizationInfo norm_utilization_out
Normalized utilization outbound for this address.
const char * GNUNET_ATS_print_preference_type(enum GNUNET_ATS_PreferenceKind type)
Convert a enum GNUNET_ATS_PreferenceType to a string.
static int address_count
Definition: gnunet-hello.c:50
struct GNUNET_TIME_Relative GNUNET_TIME_relative_subtract(struct GNUNET_TIME_Relative a1, struct GNUNET_TIME_Relative a2)
Subtract relative timestamp from the other.
Definition: time.c:602
struct RIL_Learning_Parameters parameters
Learning parameters.
GAS_solver_address_change_preference s_pref
Change relative preference for quality in solver.
#define RIL_DEFAULT_TEMPERATURE
static void envi_action_bw_double(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int direction_in)
Doubles the bandwidth for the active address.
struct GNUNET_TIME_Absolute GNUNET_TIME_absolute_get(void)
Get the current time.
Definition: time.c:118
uint32_t bw_in_utilized
Bandwidth inbound actually utilized in the network.
double social_welfare
The social welfare achieved in the scope.
int GNUNET_CONFIGURATION_get_value_string(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, char **value)
Get a configuration value that should be a string.
#define GNUNET_ATS_MaxBandwidth
Maximum bandwidth assigned to a network : 4095 MB/s.
#define RIL_DEFAULT_EXPLORE_RATIO
Address with additional information.
static struct RIL_Peer_Agent * ril_get_agent(struct GAS_RIL_Handle *solver, const struct GNUNET_PeerIdentity *peer, int create)
Returns the agent for a peer.
struct GNUNET_TESTBED_Peer * peer
The peer associated with this model.
#define GNUNET_SYSERR
Definition: gnunet_common.h:76
static void ril_calculate_discount(struct GAS_RIL_Handle *solver)
Determines how much the reward needs to be discounted depending on the amount of time, which has passed since the last time-step.
static unsigned int size
Size of the "table".
Definition: peer.c:67
void * solver_information
Solver-specific information for this address.
double ** E
Eligibility traces.
void * libgnunet_plugin_ats_ril_done(void *cls)
Exit point for the plugin.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
static struct RIL_Address_Wrapped * agent_address_get_wrapped(struct RIL_Peer_Agent *agent, struct ATS_Address *address)
Gets the wrapped address from the agent&#39;s list.
static int ril_network_count_active_agents(struct GAS_RIL_Handle *solver, struct RIL_Scope *scope)
Count the number of active agents/connections in a network scope.
unsigned int networks_count
Networks count.
struct ATS_Address * address_inuse
Address in use.
double epsilon_decay
Decay factor of the explore ratio.
struct GNUNET_MQ_Envelope * env
Definition: 005.c:1
void * cls
Closure to pass to all solver functions in this struct.
#define GNUNET_CONTAINER_DLL_insert_tail(head, tail, element)
Insert an element at the tail of a DLL.
Change the peer&#39;s latency value to the given amount.
#define RIL_DEFAULT_STEP_TIME_MIN
double epsilon_init
Initial exploration ratio value.
ats service address management
RIL_Select
#define GNUNET_TIME_UNIT_ZERO
Relative time zero.
#define GNUNET_memcmp(a, b)
Compare memory in a and b, where both must be of the same pointer type.
GAS_solver_get_preferred_address s_get
Tell solver to notify ATS if the address to use changes for a specific peer using the bandwidth chang...
unsigned long long in_quota[6]
Array of configured inbound quotas Order according to networks in network array.
A solution iteration has been finished.
#define RIL_DEFAULT_DISCOUNT_BETA
static unsigned long long ril_network_get_assigned(struct GAS_RIL_Handle *solver, enum GNUNET_NetworkType type, int direction_in)
Calculates how much bandwidth is assigned in sum in a network scope, either in the inbound or in the ...
unsigned int active_agent_count
Number of active agents in scope.
uint32_t bw_out_available
Total available outbound bandwidth.
GAS_solver_bulk_start s_bulk_start
Start a bulk operation.
struct RIL_Address_Wrapped * next
Next in DLL.
double global_discount_variable
Variable discount factor, dependent on time between steps.
The identity of the host (wraps the signing key of the peer).
unsigned int n
Number of columns of W / Number of actions.
int done
Shutdown.
static void GAS_ril_address_preference_feedback(void *solver, struct GNUNET_SERVICE_Client *application, const struct GNUNET_PeerIdentity *peer, const struct GNUNET_TIME_Relative scope, enum GNUNET_ATS_PreferenceKind kind, double score)
Give feedback about the current assignment.
uint32_t assigned_bw_out
Outbound bandwidth assigned by solver.
struct GAS_NormalizationInfo norm_utilization_in
Normalized utilization inbound for this address.
static double ril_get_used_resource_ratio(struct GAS_RIL_Handle *solver)
Determines how much of the available bandwidth is assigned.
#define RIL_ACTION_INVALID
#define RIL_NOP_DECAY
struct RIL_Peer_Agent * prev
Previous agent in solver&#39;s linked list.
#define RIL_INC_DEC_STEP_SIZE
static void agent_update(struct RIL_Peer_Agent *agent, double reward, double *s_next, int a_prime)
Updates the weights (i.e.
static void GAS_ril_bulk_start(void *solver)
Start a bulk operation.
const struct GNUNET_CONFIGURATION_Handle * cfg
Configuration handle to be used by the solver.
#define GNUNET_log(kind,...)
Entry in list of pending tasks.
Definition: scheduler.c:134
static double envi_get_reward(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent)
Gets the reward for the last performed step, which is calculated in equal parts from the local (the p...
GAS_solver_information_callback info_cb
Callback for solver to call with status information, can be NULL.
Change the peer&#39;s bandwidth value (value per byte of bandwidth in the goal function) to the given amo...
static int agent_select_action(struct RIL_Peer_Agent *agent, double *state)
Select the next action of an agent either according to the e-greedy strategy or the softmax strategy...
static void agent_step(struct RIL_Peer_Agent *agent)
Performs one step of the Markov Decision Process.
static void ril_networks_update_state(struct GAS_RIL_Handle *solver)
Retrieves the state of the network scope, so that its attributes are up-to- date. ...
struct GAS_NormalizationInfo norm_delay
Normalized delay information for this address.
unsigned long long out_quota[6]
Array of configured outbound quotas Order according to networks in network array. ...
static double agent_get_utility(struct RIL_Peer_Agent *agent)
Returns the utility value of the connection an agent manages.
double * s_old
Last perceived state feature vector.
uint32_t bw_in_assigned
Bandwidth inbound assigned in network after last step.
unsigned long long bw_out_assigned
Bandwidth outbound assigned in network after last step.
enum GNUNET_TESTBED_UnderlayLinkModelType type
the type of this model
void * cls
Closure to pass to all callbacks in this struct.
Time for absolute times used by GNUnet, in microseconds.
#define GNUNET_YES
Definition: gnunet_common.h:77
struct GNUNET_TIME_Relative GNUNET_TIME_absolute_get_difference(struct GNUNET_TIME_Absolute start, struct GNUNET_TIME_Absolute end)
Compute the time difference between the given start and end times.
Definition: time.c:353
struct RIL_Address_Wrapped * addresses_tail
Tail of addresses DLL.
static struct GNUNET_ATS_SolverFunctions * sf
Solver handle.
GNUNET_ATS_PreferenceKind
Enum defining all known preference categories.
int bulk_lock
Lock for bulk operations.
static void envi_action_bw_inc(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int direction_in)
Increases the bandwidth by 5 times the minimum bandwidth for the active address.
static struct GNUNET_ARM_Operation * op
Current operation.
Definition: gnunet-arm.c:144
The ATS plugin will pass a pointer to a struct of this type as to the initialization function of the ...
struct RIL_Peer_Agent * next
Next agent in solver&#39;s linked list.
static double envi_get_penalty(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent)
int GNUNET_CONFIGURATION_get_value_yesno(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option)
Get a configuration value that should be in a set of "YES" or "NO".
static double ril_network_get_social_welfare(struct GAS_RIL_Handle *solver, struct RIL_Scope *scope)
Calculates the social welfare within a network scope according to what social welfare measure is set ...
unsigned long long rbf_divisor
State space divisor.
struct GNUNET_TIME_Relative step_time_min
Minimal interval time between steps in milliseconds.
GAS_solver_bulk_stop s_bulk_stop
Bulk operation done.
#define LOG(kind,...)
static void ril_step(struct GAS_RIL_Handle *solver)
Prototype of the ril_step() procedure.
static char * address
GNS address for this phone.
GAS_bandwidth_changed_cb bandwidth_changed_cb
ATS addresses callback to be notified about bandwidth assignment changes.
After the problem was finished, notifications about changes to addresses are done.
#define GNUNET_TIME_UNIT_MICROSECONDS
One microsecond, our basic time unit.
static void agent_modify_eligibility(struct RIL_Peer_Agent *agent, enum RIL_E_Modification mod, double *feature, int action)
Changes the eligibility trace vector e in various manners: RIL_E_ACCUMULATE - adds feature to each co...
struct ATS_Address * suggestion_address
The address which has to be issued.
struct GNUNET_TIME_Relative GNUNET_TIME_relative_saturating_multiply(struct GNUNET_TIME_Relative rel, unsigned long long factor)
Saturating multiply relative time by a given factor.
Definition: time.c:499
const char * GNUNET_i2s(const struct GNUNET_PeerIdentity *pid)
Convert a peer identity to a string (for printing debug messages).
double norm
Normalized values from queue to a range of values [1.0...2.0].
static int agent_action_is_possible(struct RIL_Peer_Agent *agent, int action)
struct GAS_RIL_Handle * envi
Environment handle.
unsigned int network_count
Number of networks available, size of the out_quota and in_quota arrays.
RIL_Algorithm
Wrapper for addresses to store them in agent&#39;s linked list.
No good quality of the operation is needed (i.e., random numbers can be pseudo-random).
#define GNUNET_malloc(size)
Wrapper around malloc.
double temperature_init
Initial softmax action-selection temperature.
#define GNUNET_free(ptr)
Wrapper around free.
Time for relative time used by GNUnet, in microseconds.
double nop_bonus
NOP bonus.
double objective_old
The agent&#39;s last objective value.
double alpha
Gradient-descent step-size.
uint32_t bw_in_available
Total available inbound bandwidth.
unsigned long long step_count
Number of performed time-steps.
void * GNUNET_SCHEDULER_cancel(struct GNUNET_SCHEDULER_Task *task)
Cancel the task with the specified identifier.
Definition: scheduler.c:966
#define RIL_MIN_BW
enum RIL_E_Modification eligibility_trace_mode
Whether to accumulate or replace eligibility traces.
RIL_Action_Type
ATS reinforcement learning solver.