GNUnet  0.10.x
Data Structures | Macros | Enumerations | Functions
plugin_ats_ril.c File Reference

ATS reinforcement learning solver. More...

#include "platform.h"
#include <float.h>
#include <math.h>
#include "gnunet_ats_plugin.h"
#include "gnunet-service-ats_addresses.h"
Include dependency graph for plugin_ats_ril.c:

Go to the source code of this file.

Data Structures

struct  RIL_Learning_Parameters
 Global learning parameters. More...
 
struct  RIL_Address_Wrapped
 Wrapper for addresses to store them in agent's linked list. More...
 
struct  RIL_Peer_Agent
 
struct  RIL_Scope
 
struct  GAS_RIL_Handle
 A handle for the reinforcement learning solver. More...
 

Macros

#define LOG(kind, ...)   GNUNET_log_from (kind, "ats-ril", __VA_ARGS__)
 
#define RIL_MIN_BW
 
#define RIL_MAX_BW   GNUNET_ATS_MaxBandwidth
 
#define RIL_ACTION_INVALID   -1
 
#define RIL_INTERVAL_EXPONENT   10
 
#define RIL_UTILITY_DELAY_MAX   1000
 
#define RIL_DEFAULT_STEP_TIME_MIN
 
#define RIL_DEFAULT_STEP_TIME_MAX
 
#define RIL_DEFAULT_ALGORITHM   RIL_ALGO_SARSA
 
#define RIL_DEFAULT_SELECT   RIL_SELECT_SOFTMAX
 
#define RIL_DEFAULT_WELFARE   RIL_WELFARE_NASH
 
#define RIL_DEFAULT_DISCOUNT_BETA   0.6
 
#define RIL_DEFAULT_DISCOUNT_GAMMA   0.5
 
#define RIL_DEFAULT_GRADIENT_STEP_SIZE   0.01
 
#define RIL_DEFAULT_TRACE_DECAY   0.5
 
#define RIL_DEFAULT_EXPLORE_RATIO   1
 
#define RIL_DEFAULT_EXPLORE_DECAY   0.95
 
#define RIL_DEFAULT_RBF_DIVISOR   50
 
#define RIL_DEFAULT_TEMPERATURE   0.1
 
#define RIL_DEFAULT_TEMPERATURE_DECAY   1
 
#define RIL_INC_DEC_STEP_SIZE   1
 
#define RIL_NOP_DECAY   0.5
 

Enumerations

enum  RIL_Action_Type {
  RIL_ACTION_NOTHING = 0, RIL_ACTION_BW_IN_DBL = -2, RIL_ACTION_BW_IN_HLV = -3, RIL_ACTION_BW_IN_INC = 1,
  RIL_ACTION_BW_IN_DEC = 2, RIL_ACTION_BW_OUT_DBL = -4, RIL_ACTION_BW_OUT_HLV = -5, RIL_ACTION_BW_OUT_INC = 3,
  RIL_ACTION_BW_OUT_DEC = 4, RIL_ACTION_TYPE_NUM = 5
}
 ATS reinforcement learning solver. More...
 
enum  RIL_Algorithm { RIL_ALGO_SARSA = 0, RIL_ALGO_Q = 1 }
 
enum  RIL_Select { RIL_SELECT_SOFTMAX = 0, RIL_SELECT_EGREEDY = 1 }
 
enum  RIL_Welfare { RIL_WELFARE_NASH, RIL_WELFARE_EGALITARIAN }
 
enum  RIL_E_Modification { RIL_E_DECAY, RIL_E_ZERO, RIL_E_ACCUMULATE, RIL_E_REPLACE }
 

Functions

static double agent_q (struct RIL_Peer_Agent *agent, const double *state, int action)
 Estimate the current action-value for state s and action a. More...
 
static int agent_address_get_index (struct RIL_Peer_Agent *agent, struct ATS_Address *address)
 Get the index of the address in the agent's list. More...
 
static struct RIL_Address_Wrappedagent_address_get_wrapped (struct RIL_Peer_Agent *agent, struct ATS_Address *address)
 Gets the wrapped address from the agent's list. More...
 
static int agent_action_is_possible (struct RIL_Peer_Agent *agent, int action)
 
static int agent_get_action_max (struct RIL_Peer_Agent *agent, double *state)
 Gets the action, with the maximal estimated Q-value (i.e. More...
 
static int agent_get_action_random (struct RIL_Peer_Agent *agent)
 Chooses a random action from the set of possible ones. More...
 
static void agent_update (struct RIL_Peer_Agent *agent, double reward, double *s_next, int a_prime)
 Updates the weights (i.e. More...
 
static void agent_modify_eligibility (struct RIL_Peer_Agent *agent, enum RIL_E_Modification mod, double *feature, int action)
 Changes the eligibility trace vector e in various manners: RIL_E_ACCUMULATE - adds feature to each component as in accumulating eligibility traces RIL_E_REPLACE - resets each component to feature as in replacing traces RIL_E_DECAY - multiplies e with discount factor and lambda as in the update rule RIL_E_ZERO - sets e to 0 as in Watkin's Q-learning algorithm when exploring and when initializing. More...
 
static void ril_inform (struct GAS_RIL_Handle *solver, enum GAS_Solver_Operation op, enum GAS_Solver_Status stat)
 Informs the environment about the status of the solver. More...
 
static unsigned long long ril_get_max_bw (struct RIL_Scope *net)
 Calculates the maximum bandwidth an agent can assign in a network scope. More...
 
static void envi_set_active_suggestion (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, struct ATS_Address *new_address, unsigned long long new_bw_in, unsigned long long new_bw_out, int silent)
 Changes the active assignment suggestion of the handler and invokes the bw_changed callback to notify ATS of its new decision. More...
 
static double * envi_get_state (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent)
 Allocates a state vector and fills it with the features present. More...
 
static double agent_get_utility (struct RIL_Peer_Agent *agent)
 Returns the utility value of the connection an agent manages. More...
 
static double ril_network_get_social_welfare (struct GAS_RIL_Handle *solver, struct RIL_Scope *scope)
 Calculates the social welfare within a network scope according to what social welfare measure is set in the configuration. More...
 
static double envi_get_penalty (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent)
 
static double envi_get_reward (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent)
 Gets the reward for the last performed step, which is calculated in equal parts from the local (the peer specific) and the global (for all peers identical) reward. More...
 
static void envi_action_bw_double (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int direction_in)
 Doubles the bandwidth for the active address. More...
 
static void envi_action_bw_halven (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int direction_in)
 Cuts the bandwidth for the active address in half. More...
 
static void envi_action_bw_inc (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int direction_in)
 Increases the bandwidth by 5 times the minimum bandwidth for the active address. More...
 
static void envi_action_bw_dec (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int direction_in)
 Decreases the bandwidth by 5 times the minimum bandwidth for the active address. More...
 
static void envi_action_address_switch (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, unsigned int address_index)
 Switches to the address given by its index. More...
 
static void envi_do_action (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int action)
 Puts the action into effect by calling the according function. More...
 
static int agent_select_egreedy (struct RIL_Peer_Agent *agent, double *state)
 Selects the next action using the e-greedy strategy. More...
 
static int agent_select_softmax (struct RIL_Peer_Agent *agent, double *state)
 Selects the next action with a probability corresponding to its value. More...
 
static int agent_select_action (struct RIL_Peer_Agent *agent, double *state)
 Select the next action of an agent either according to the e-greedy strategy or the softmax strategy. More...
 
static void agent_step (struct RIL_Peer_Agent *agent)
 Performs one step of the Markov Decision Process. More...
 
static void ril_step (struct GAS_RIL_Handle *solver)
 Prototype of the ril_step() procedure. More...
 
static void ril_step_scheduler_task (void *cls)
 Task for the scheduler, which performs one step and lets the solver know that no further step is scheduled. More...
 
static double ril_get_used_resource_ratio (struct GAS_RIL_Handle *solver)
 Determines how much of the available bandwidth is assigned. More...
 
static struct RIL_Scoperil_get_network (struct GAS_RIL_Handle *s, uint32_t type)
 Lookup network struct by type. More...
 
static int ril_network_is_not_full (struct GAS_RIL_Handle *solver, enum GNUNET_NetworkType network)
 Determines whether more connections are allocated in a network scope, than they would theoretically fit. More...
 
static void ril_try_unblock_agent (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int silent)
 Unblocks an agent for which a connection request is there, that could not be satisfied. More...
 
static void ril_calculate_discount (struct GAS_RIL_Handle *solver)
 Determines how much the reward needs to be discounted depending on the amount of time, which has passed since the last time-step. More...
 
static int ril_network_count_active_agents (struct GAS_RIL_Handle *solver, struct RIL_Scope *scope)
 Count the number of active agents/connections in a network scope. More...
 
static unsigned long long ril_network_get_assigned (struct GAS_RIL_Handle *solver, enum GNUNET_NetworkType type, int direction_in)
 Calculates how much bandwidth is assigned in sum in a network scope, either in the inbound or in the outbound direction. More...
 
static unsigned long long ril_network_get_utilized (struct GAS_RIL_Handle *solver, enum GNUNET_NetworkType type, int direction_in)
 Calculates how much bandwidth is actually utilized in sum in a network scope, either in the inbound or in the outbound direction. More...
 
static void ril_networks_update_state (struct GAS_RIL_Handle *solver)
 Retrieves the state of the network scope, so that its attributes are up-to- date. More...
 
static void ril_step_schedule_next (struct GAS_RIL_Handle *solver)
 Schedules the next global step in an adaptive way. More...
 
static void agent_w_init (struct RIL_Peer_Agent *agent)
 Initializes the matrix W of parameter vectors theta with small random numbers. More...
 
static struct RIL_Peer_Agentagent_init (void *s, const struct GNUNET_PeerIdentity *peer)
 Initialize an agent without addresses and its knowledge base. More...
 
static void agent_die (struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent)
 Deallocate agent. More...
 
static struct RIL_Peer_Agentril_get_agent (struct GAS_RIL_Handle *solver, const struct GNUNET_PeerIdentity *peer, int create)
 Returns the agent for a peer. More...
 
static int ril_network_is_active (struct GAS_RIL_Handle *solver, enum GNUNET_NetworkType network)
 Determine whether at least the minimum bandwidth is set for the network. More...
 
static void ril_cut_from_vector (void **old, size_t element_size, unsigned int hole_start, unsigned int hole_length, unsigned int old_length)
 Cuts a slice out of a vector of elements. More...
 
static void GAS_ril_address_change_preference (void *solver, const struct GNUNET_PeerIdentity *peer, enum GNUNET_ATS_PreferenceKind kind, double pref_rel)
 Change relative preference for quality in solver. More...
 
static void GAS_ril_address_add (void *solver, struct ATS_Address *address, uint32_t network)
 Add a new address for a peer to the solver. More...
 
static void GAS_ril_address_delete (void *solver, struct ATS_Address *address)
 Delete an address in the solver. More...
 
static void GAS_ril_address_property_changed (void *solver, struct ATS_Address *address)
 Update the properties of an address in the solver. More...
 
static void GAS_ril_address_preference_feedback (void *solver, struct GNUNET_SERVICE_Client *application, const struct GNUNET_PeerIdentity *peer, const struct GNUNET_TIME_Relative scope, enum GNUNET_ATS_PreferenceKind kind, double score)
 Give feedback about the current assignment. More...
 
static void GAS_ril_bulk_start (void *solver)
 Start a bulk operation. More...
 
static void GAS_ril_bulk_stop (void *solver)
 Bulk operation done. More...
 
static void GAS_ril_get_preferred_address (void *solver, const struct GNUNET_PeerIdentity *peer)
 Tell solver to notify ATS if the address to use changes for a specific peer using the bandwidth changed callback. More...
 
static void GAS_ril_stop_get_preferred_address (void *solver, const struct GNUNET_PeerIdentity *peer)
 Tell solver stop notifying ATS about changes for this peers. More...
 
void * libgnunet_plugin_ats_ril_init (void *cls)
 Entry point for the plugin. More...
 
void * libgnunet_plugin_ats_ril_done (void *cls)
 Exit point for the plugin. More...
 

Detailed Description

ATS reinforcement learning solver.

Author
Fabian Oehlmann
Matthias Wachs

Definition in file plugin_ats_ril.c.

Macro Definition Documentation

◆ LOG

#define LOG (   kind,
  ... 
)    GNUNET_log_from (kind, "ats-ril", __VA_ARGS__)

◆ RIL_MIN_BW

#define RIL_MIN_BW
Value:
(5 * ntohl ( \
value__))
#define GNUNET_CONSTANTS_DEFAULT_BW_IN_OUT
Bandwidth (in/out) to assume initially (before either peer has communicated any particular preference...

Definition at line 37 of file plugin_ats_ril.c.

Referenced by agent_get_utility(), agent_init(), envi_action_bw_dec(), envi_action_bw_inc(), envi_get_penalty(), ril_network_is_active(), ril_network_is_not_full(), and ril_try_unblock_agent().

◆ RIL_MAX_BW

#define RIL_MAX_BW   GNUNET_ATS_MaxBandwidth

Definition at line 40 of file plugin_ats_ril.c.

Referenced by agent_action_is_possible().

◆ RIL_ACTION_INVALID

#define RIL_ACTION_INVALID   -1

◆ RIL_INTERVAL_EXPONENT

#define RIL_INTERVAL_EXPONENT   10

Definition at line 43 of file plugin_ats_ril.c.

Referenced by ril_step_schedule_next().

◆ RIL_UTILITY_DELAY_MAX

#define RIL_UTILITY_DELAY_MAX   1000

Definition at line 44 of file plugin_ats_ril.c.

Referenced by agent_get_utility().

◆ RIL_DEFAULT_STEP_TIME_MIN

#define RIL_DEFAULT_STEP_TIME_MIN
Value:
struct GNUNET_TIME_Relative GNUNET_TIME_relative_multiply(struct GNUNET_TIME_Relative rel, unsigned long long factor)
Multiply relative time by a given factor.
Definition: time.c:440
#define GNUNET_TIME_UNIT_MILLISECONDS
One millisecond.

Definition at line 46 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_STEP_TIME_MAX

#define RIL_DEFAULT_STEP_TIME_MAX
Value:
struct GNUNET_TIME_Relative GNUNET_TIME_relative_multiply(struct GNUNET_TIME_Relative rel, unsigned long long factor)
Multiply relative time by a given factor.
Definition: time.c:440
#define GNUNET_TIME_UNIT_MILLISECONDS
One millisecond.

Definition at line 48 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_ALGORITHM

#define RIL_DEFAULT_ALGORITHM   RIL_ALGO_SARSA

Definition at line 50 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_SELECT

#define RIL_DEFAULT_SELECT   RIL_SELECT_SOFTMAX

Definition at line 51 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_WELFARE

#define RIL_DEFAULT_WELFARE   RIL_WELFARE_NASH

Definition at line 52 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_DISCOUNT_BETA

#define RIL_DEFAULT_DISCOUNT_BETA   0.6

Definition at line 53 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_DISCOUNT_GAMMA

#define RIL_DEFAULT_DISCOUNT_GAMMA   0.5

Definition at line 54 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_GRADIENT_STEP_SIZE

#define RIL_DEFAULT_GRADIENT_STEP_SIZE   0.01

Definition at line 55 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_TRACE_DECAY

#define RIL_DEFAULT_TRACE_DECAY   0.5

Definition at line 56 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_EXPLORE_RATIO

#define RIL_DEFAULT_EXPLORE_RATIO   1

Definition at line 57 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_EXPLORE_DECAY

#define RIL_DEFAULT_EXPLORE_DECAY   0.95

Definition at line 58 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_RBF_DIVISOR

#define RIL_DEFAULT_RBF_DIVISOR   50

Definition at line 59 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_TEMPERATURE

#define RIL_DEFAULT_TEMPERATURE   0.1

Definition at line 60 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_DEFAULT_TEMPERATURE_DECAY

#define RIL_DEFAULT_TEMPERATURE_DECAY   1

Definition at line 61 of file plugin_ats_ril.c.

Referenced by libgnunet_plugin_ats_ril_init().

◆ RIL_INC_DEC_STEP_SIZE

#define RIL_INC_DEC_STEP_SIZE   1

Definition at line 63 of file plugin_ats_ril.c.

Referenced by envi_action_bw_dec(), and envi_action_bw_inc().

◆ RIL_NOP_DECAY

#define RIL_NOP_DECAY   0.5

Definition at line 64 of file plugin_ats_ril.c.

Referenced by envi_get_reward().

Enumeration Type Documentation

◆ RIL_Action_Type

ATS reinforcement learning solver.

General description The actions, how an agent can manipulate the current assignment. I.e. how the bandwidth can be changed for the currently chosen address. Not depicted in the enum are the actions of switching to a particular address. The action of switching to address with index i is depicted by the number (RIL_ACTION_TYPE_NUM + i).

Enumerator
RIL_ACTION_NOTHING 
RIL_ACTION_BW_IN_DBL 
RIL_ACTION_BW_IN_HLV 
RIL_ACTION_BW_IN_INC 
RIL_ACTION_BW_IN_DEC 
RIL_ACTION_BW_OUT_DBL 
RIL_ACTION_BW_OUT_HLV 
RIL_ACTION_BW_OUT_INC 
RIL_ACTION_BW_OUT_DEC 
RIL_ACTION_TYPE_NUM 

Definition at line 78 of file plugin_ats_ril.c.

◆ RIL_Algorithm

Enumerator
RIL_ALGO_SARSA 
RIL_ALGO_Q 

Definition at line 92 of file plugin_ats_ril.c.

93 {
94  RIL_ALGO_SARSA = 0,
95  RIL_ALGO_Q = 1
96 };

◆ RIL_Select

enum RIL_Select
Enumerator
RIL_SELECT_SOFTMAX 
RIL_SELECT_EGREEDY 

Definition at line 98 of file plugin_ats_ril.c.

◆ RIL_Welfare

Enumerator
RIL_WELFARE_NASH 
RIL_WELFARE_EGALITARIAN 

Definition at line 104 of file plugin_ats_ril.c.

◆ RIL_E_Modification

Enumerator
RIL_E_DECAY 
RIL_E_ZERO 
RIL_E_ACCUMULATE 
RIL_E_REPLACE 

Definition at line 110 of file plugin_ats_ril.c.

Function Documentation

◆ agent_q()

static double agent_q ( struct RIL_Peer_Agent agent,
const double *  state,
int  action 
)
static

Estimate the current action-value for state s and action a.

Parameters
agentagent performing the estimation
states
actiona
Returns
estimation value

Definition at line 483 of file plugin_ats_ril.c.

References RIL_Peer_Agent::m, result, and RIL_Peer_Agent::W.

Referenced by agent_get_action_max(), agent_select_softmax(), and agent_update().

486 {
487  unsigned int i;
488  double result = 0.0;
489 
490  for (i = 0; i < agent->m; i++)
491  result += state[i] * agent->W[action][i];
492 
493  /* prevent crashes if learning diverges */
494  if (isnan (result))
495  return isnan (result) * UINT32_MAX;
496  if (isinf (result))
497  return isinf (result) * UINT32_MAX;
498  return result;
499 }
unsigned int m
Number of rows of W / Number of state-vector features.
enum State state
current state of profiling
static int result
Global testing status.
double ** W
Experience matrix W.
Here is the caller graph for this function:

◆ agent_address_get_index()

static int agent_address_get_index ( struct RIL_Peer_Agent agent,
struct ATS_Address address 
)
static

Get the index of the address in the agent's list.

Parameters
agentagent handle
addressaddress handle
Returns
the index, starting with zero

Definition at line 510 of file plugin_ats_ril.c.

References RIL_Address_Wrapped::address_naked, RIL_Peer_Agent::addresses_head, and RIL_Address_Wrapped::next.

Referenced by agent_action_is_possible(), envi_do_action(), envi_get_state(), and GAS_ril_address_delete().

512 {
513  int i;
514  struct RIL_Address_Wrapped *cur;
515 
516  i = -1;
517  for (cur = agent->addresses_head; NULL != cur; cur = cur->next)
518  {
519  i++;
520  if (cur->address_naked == address)
521  return i;
522  }
523  return i;
524 }
struct RIL_Address_Wrapped * addresses_head
Head of addresses DLL.
struct ATS_Address * address_naked
The address.
struct RIL_Address_Wrapped * next
Next in DLL.
Wrapper for addresses to store them in agent&#39;s linked list.
Here is the caller graph for this function:

◆ agent_address_get_wrapped()

static struct RIL_Address_Wrapped* agent_address_get_wrapped ( struct RIL_Peer_Agent agent,
struct ATS_Address address 
)
static

Gets the wrapped address from the agent's list.

Parameters
agentagent handle
addressaddress handle
Returns
wrapped address

Definition at line 535 of file plugin_ats_ril.c.

References RIL_Address_Wrapped::address_naked, RIL_Peer_Agent::addresses_head, and RIL_Address_Wrapped::next.

Referenced by GAS_ril_address_delete().

537 {
538  struct RIL_Address_Wrapped *cur;
539 
540  for (cur = agent->addresses_head; NULL != cur; cur = cur->next)
541  if (cur->address_naked == address)
542  return cur;
543  return NULL;
544 }
struct RIL_Address_Wrapped * addresses_head
Head of addresses DLL.
struct ATS_Address * address_naked
The address.
struct RIL_Address_Wrapped * next
Next in DLL.
Wrapper for addresses to store them in agent&#39;s linked list.
Here is the caller graph for this function:

◆ agent_action_is_possible()

static int agent_action_is_possible ( struct RIL_Peer_Agent agent,
int  action 
)
static

Definition at line 548 of file plugin_ats_ril.c.

References ATS_Address::active, RIL_Peer_Agent::address_inuse, RIL_Address_Wrapped::address_naked, RIL_Peer_Agent::addresses_tail, agent_address_get_index(), RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, GNUNET_assert, GNUNET_NO, GNUNET_YES, RIL_ACTION_BW_IN_DBL, RIL_ACTION_BW_IN_DEC, RIL_ACTION_BW_IN_HLV, RIL_ACTION_BW_IN_INC, RIL_ACTION_BW_OUT_DBL, RIL_ACTION_BW_OUT_DEC, RIL_ACTION_BW_OUT_HLV, RIL_ACTION_BW_OUT_INC, RIL_ACTION_NOTHING, RIL_ACTION_TYPE_NUM, and RIL_MAX_BW.

Referenced by agent_get_action_max(), agent_get_action_random(), and agent_select_softmax().

549 {
550  int address_index;
551 
552  switch (action)
553  {
554  case RIL_ACTION_NOTHING:
555  return GNUNET_YES;
556  break;
557 
560  if (agent->bw_in >= RIL_MAX_BW)
561  return GNUNET_NO;
562  else
563  return GNUNET_YES;
564  break;
565 
568  if (agent->bw_in <= 0)
569  return GNUNET_NO;
570  else
571  return GNUNET_YES;
572  break;
573 
576  if (agent->bw_out >= RIL_MAX_BW)
577  return GNUNET_NO;
578  else
579  return GNUNET_YES;
580  break;
581 
584  if (agent->bw_out <= 0)
585  return GNUNET_NO;
586  else
587  return GNUNET_YES;
588  break;
589 
590  default:
591  if ((action >= RIL_ACTION_TYPE_NUM) && (action < agent->n)) // switch address action
592  {
593  address_index = action - RIL_ACTION_TYPE_NUM;
594 
595  GNUNET_assert (address_index >= 0);
596  GNUNET_assert (
597  address_index <= agent_address_get_index (agent,
598  agent->addresses_tail->
599  address_naked));
600 
601  if ((agent_address_get_index (agent, agent->address_inuse) ==
602  address_index) ||
603  agent->address_inuse->active)
604  return GNUNET_NO;
605  else
606  return GNUNET_YES;
607  break;
608  }
609  // error - action does not exist
611  }
612 }
#define RIL_MAX_BW
uint32_t bw_in
Inbound bandwidth assigned by the agent.
int active
Is this the active address for this peer?
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
#define GNUNET_NO
Definition: gnunet_common.h:78
struct ATS_Address * address_naked
The address.
static int agent_address_get_index(struct RIL_Peer_Agent *agent, struct ATS_Address *address)
Get the index of the address in the agent&#39;s list.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
#define GNUNET_YES
Definition: gnunet_common.h:77
struct RIL_Address_Wrapped * addresses_tail
Tail of addresses DLL.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_get_action_max()

static int agent_get_action_max ( struct RIL_Peer_Agent agent,
double *  state 
)
static

Gets the action, with the maximal estimated Q-value (i.e.

the one currently estimated to bring the most reward in the future)

Parameters
agentagent performing the calculation
statethe state from which to take the action
Returns
the action promising most future reward

Definition at line 624 of file plugin_ats_ril.c.

References agent_action_is_possible(), agent_q(), GNUNET_assert, RIL_Peer_Agent::n, and RIL_ACTION_INVALID.

Referenced by agent_select_egreedy(), agent_select_softmax(), and agent_step().

625 {
626  int i;
627  int max_i = RIL_ACTION_INVALID;
628  double cur_q;
629  double max_q = -DBL_MAX;
630 
631  for (i = 0; i < agent->n; i++)
632  {
633  if (agent_action_is_possible (agent, i))
634  {
635  cur_q = agent_q (agent, state, i);
636  if (cur_q > max_q)
637  {
638  max_q = cur_q;
639  max_i = i;
640  }
641  }
642  }
643 
645 
646  return max_i;
647 }
static double agent_q(struct RIL_Peer_Agent *agent, const double *state, int action)
Estimate the current action-value for state s and action a.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
enum State state
current state of profiling
unsigned int n
Number of columns of W / Number of actions.
#define RIL_ACTION_INVALID
static int agent_action_is_possible(struct RIL_Peer_Agent *agent, int action)
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_get_action_random()

static int agent_get_action_random ( struct RIL_Peer_Agent agent)
static

Chooses a random action from the set of possible ones.

Parameters
agentthe agent performing the action
Returns
the action index

Definition at line 656 of file plugin_ats_ril.c.

References agent_action_is_possible(), GNUNET_assert, GNUNET_CRYPTO_QUALITY_WEAK, GNUNET_CRYPTO_random_u32(), GNUNET_NO, GNUNET_YES, RIL_Peer_Agent::n, and RIL_ACTION_INVALID.

Referenced by agent_select_egreedy().

657 {
658  int i;
659  int is_possible[agent->n];
660  int sum = 0;
661  int r;
662 
663  for (i = 0; i < agent->n; i++)
664  {
665  if (agent_action_is_possible (agent, i))
666  {
667  is_possible[i] = GNUNET_YES;
668  sum++;
669  }
670  else
671  {
672  is_possible[i] = GNUNET_NO;
673  }
674  }
675 
677 
678  sum = -1;
679  for (i = 0; i < agent->n; i++)
680  {
681  if (is_possible[i])
682  {
683  sum++;
684  if (sum == r)
685  return i;
686  }
687  }
688 
690  return RIL_ACTION_INVALID;
691 }
uint32_t GNUNET_CRYPTO_random_u32(enum GNUNET_CRYPTO_Quality mode, uint32_t i)
Produce a random value.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
#define GNUNET_NO
Definition: gnunet_common.h:78
unsigned int n
Number of columns of W / Number of actions.
#define RIL_ACTION_INVALID
#define GNUNET_YES
Definition: gnunet_common.h:77
static int agent_action_is_possible(struct RIL_Peer_Agent *agent, int action)
No good quality of the operation is needed (i.e., random numbers can be pseudo-random).
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_update()

static void agent_update ( struct RIL_Peer_Agent agent,
double  reward,
double *  s_next,
int  a_prime 
)
static

Updates the weights (i.e.

coefficients) of the weight vector in matrix W for action a

Parameters
agentthe agent performing the update
rewardthe reward received for the last action
s_nextthe new state, the last step got the agent into
a_primethe new

Definition at line 703 of file plugin_ats_ril.c.

References RIL_Peer_Agent::a_old, agent_q(), RIL_Learning_Parameters::alpha, delta, RIL_Peer_Agent::E, RIL_Peer_Agent::envi, GAS_RIL_Handle::global_discount_integrated, GAS_RIL_Handle::global_discount_variable, RIL_Peer_Agent::m, RIL_Peer_Agent::n, GAS_RIL_Handle::parameters, RIL_Peer_Agent::s_old, and RIL_Peer_Agent::W.

Referenced by agent_step().

705 {
706  int i;
707  int k;
708  double delta;
709  double **theta = agent->W;
710 
711  delta = agent->envi->global_discount_integrated * reward; // reward
712  delta += agent->envi->global_discount_variable * agent_q (agent, s_next,
713  a_prime); // discounted future value
714  delta -= agent_q (agent, agent->s_old, agent->a_old); // one step
715 
716 // LOG(GNUNET_ERROR_TYPE_INFO, "update() Step# %llu Q(s,a): %f a: %f r: %f y: %f Q(s+1,a+1) = %f delta: %f\n",
717 // agent->step_count,
718 // agent_q (agent, agent->s_old, agent->a_old),
719 // agent->envi->parameters.alpha,
720 // reward,
721 // agent->envi->global_discount_variable,
722 // agent_q (agent, s_next, a_prime),
723 // delta);
724 
725  for (k = 0; k < agent->n; k++)
726  {
727  for (i = 0; i < agent->m; i++)
728  {
729  // LOG(GNUNET_ERROR_TYPE_INFO, "alpha = %f delta = %f e[%d] = %f\n",
730  // agent->envi->parameters.alpha,
731  // delta,
732  // i,
733  // agent->e[i]);
734  theta[k][i] += agent->envi->parameters.alpha * delta * agent->E[k][i];
735  }
736  }
737 }
double global_discount_integrated
Integrated variable discount factor, dependent on time between steps.
static struct GNUNET_TIME_Relative delta
Definition: speedup.c:35
unsigned int m
Number of rows of W / Number of state-vector features.
static double agent_q(struct RIL_Peer_Agent *agent, const double *state, int action)
Estimate the current action-value for state s and action a.
int a_old
Last chosen action.
double ** W
Experience matrix W.
struct RIL_Learning_Parameters parameters
Learning parameters.
double ** E
Eligibility traces.
double global_discount_variable
Variable discount factor, dependent on time between steps.
unsigned int n
Number of columns of W / Number of actions.
double * s_old
Last perceived state feature vector.
struct GAS_RIL_Handle * envi
Environment handle.
double alpha
Gradient-descent step-size.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_modify_eligibility()

static void agent_modify_eligibility ( struct RIL_Peer_Agent agent,
enum RIL_E_Modification  mod,
double *  feature,
int  action 
)
static

Changes the eligibility trace vector e in various manners: RIL_E_ACCUMULATE - adds feature to each component as in accumulating eligibility traces RIL_E_REPLACE - resets each component to feature as in replacing traces RIL_E_DECAY - multiplies e with discount factor and lambda as in the update rule RIL_E_ZERO - sets e to 0 as in Watkin's Q-learning algorithm when exploring and when initializing.

Parameters
agentthe agent handle
modthe kind of modification
featurethe feature vector
actionthe action to take

Definition at line 753 of file plugin_ats_ril.c.

References RIL_Peer_Agent::E, RIL_Peer_Agent::envi, GAS_RIL_Handle::global_discount_variable, RIL_Learning_Parameters::lambda, RIL_Peer_Agent::m, RIL_Peer_Agent::n, GAS_RIL_Handle::parameters, RIL_E_ACCUMULATE, RIL_E_DECAY, RIL_E_REPLACE, and RIL_E_ZERO.

Referenced by agent_step().

757 {
758  int i;
759  int k;
760 
761  for (i = 0; i < agent->m; i++)
762  {
763  switch (mod)
764  {
765  case RIL_E_ACCUMULATE:
766  agent->E[action][i] += feature[i];
767  break;
768 
769  case RIL_E_REPLACE:
770  agent->E[action][i] = agent->E[action][i] > feature[i] ?
771  agent->E[action][i] : feature[i];
772  break;
773 
774  case RIL_E_DECAY:
775  for (k = 0; k < agent->n; k++)
776  {
777  agent->E[k][i] *= agent->envi->global_discount_variable
778  * agent->envi->parameters.lambda;
779  }
780  break;
781 
782  case RIL_E_ZERO:
783  for (k = 0; k < agent->n; k++)
784  {
785  agent->E[k][i] = 0;
786  }
787  break;
788  }
789  }
790 }
double lambda
Trace-decay factor for eligibility traces.
unsigned int m
Number of rows of W / Number of state-vector features.
struct RIL_Learning_Parameters parameters
Learning parameters.
double ** E
Eligibility traces.
double global_discount_variable
Variable discount factor, dependent on time between steps.
unsigned int n
Number of columns of W / Number of actions.
struct GAS_RIL_Handle * envi
Environment handle.
Here is the caller graph for this function:

◆ ril_inform()

static void ril_inform ( struct GAS_RIL_Handle solver,
enum GAS_Solver_Operation  op,
enum GAS_Solver_Status  stat 
)
static

Informs the environment about the status of the solver.

Parameters
solver
op
stat

Definition at line 800 of file plugin_ats_ril.c.

References GNUNET_ATS_PluginEnvironment::cls, GAS_RIL_Handle::env, GAS_INFO_NONE, and GNUNET_ATS_PluginEnvironment::info_cb.

Referenced by ril_step().

803 {
804  solver->env->info_cb (solver->env->cls,
805  op,
806  stat,
807  GAS_INFO_NONE);
808 }
struct GNUNET_ATS_PluginEnvironment * env
The solver-plugin environment of the solver-plugin API.
No more specific information.
GAS_solver_information_callback info_cb
Callback for solver to call with status information, can be NULL.
void * cls
Closure to pass to all callbacks in this struct.
static struct GNUNET_ARM_Operation * op
Current operation.
Definition: gnunet-arm.c:144
Here is the caller graph for this function:

◆ ril_get_max_bw()

static unsigned long long ril_get_max_bw ( struct RIL_Scope net)
static

Calculates the maximum bandwidth an agent can assign in a network scope.

Parameters
net

Definition at line 816 of file plugin_ats_ril.c.

References RIL_Scope::bw_in_available, RIL_Scope::bw_out_available, GNUNET_ATS_MaxBandwidth, GNUNET_MAX, and GNUNET_MIN.

Referenced by envi_action_bw_double(), envi_action_bw_inc(), and envi_get_state().

817 {
818  return GNUNET_MIN (2 * GNUNET_MAX (net->bw_in_available,
819  net->bw_out_available),
821 }
#define GNUNET_MAX(a, b)
Definition: gnunet_common.h:82
#define GNUNET_MIN(a, b)
Definition: gnunet_common.h:80
#define GNUNET_ATS_MaxBandwidth
Maximum bandwidth assigned to a network : 4095 MB/s.
uint32_t bw_out_available
Total available outbound bandwidth.
uint32_t bw_in_available
Total available inbound bandwidth.
Here is the caller graph for this function:

◆ envi_set_active_suggestion()

static void envi_set_active_suggestion ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent,
struct ATS_Address new_address,
unsigned long long  new_bw_in,
unsigned long long  new_bw_out,
int  silent 
)
static

Changes the active assignment suggestion of the handler and invokes the bw_changed callback to notify ATS of its new decision.

Parameters
solversolver handle
agentagent handle
new_addressthe address which is to be used
new_bw_inthe new amount of inbound bandwidth set for this address
new_bw_outthe new amount of outbound bandwidth set for this address
silentdisables invocation of the bw_changed callback, if GNUNET_YES

Definition at line 835 of file plugin_ats_ril.c.

References ATS_Address::active, RIL_Peer_Agent::address_inuse, ATS_Address::assigned_bw_in, ATS_Address::assigned_bw_out, RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, GNUNET_assert, GNUNET_ERROR_TYPE_DEBUG, GNUNET_i2s(), GNUNET_NO, GNUNET_YES, RIL_Peer_Agent::is_active, LOG, notify, RIL_Peer_Agent::peer, RIL_Peer_Agent::suggestion_address, and RIL_Peer_Agent::suggestion_issue.

Referenced by envi_action_address_switch(), envi_action_bw_dec(), envi_action_bw_double(), envi_action_bw_halven(), envi_action_bw_inc(), GAS_ril_address_delete(), GAS_ril_get_preferred_address(), GAS_ril_stop_get_preferred_address(), and ril_try_unblock_agent().

841 {
842  int notify = GNUNET_NO;
843 
845  " set_active_suggestion() for peer '%s'\n",
846  GNUNET_i2s (&agent->peer));
847 
848  // address change
849  if (agent->address_inuse != new_address)
850  {
851  if (NULL != agent->address_inuse)
852  {
853  agent->address_inuse->active = GNUNET_NO;
854  agent->address_inuse->assigned_bw_in = 0;
855  agent->address_inuse->assigned_bw_out = 0;
856  }
857  if (NULL != new_address)
858  {
859  LOG (GNUNET_ERROR_TYPE_DEBUG, " set address active: %s\n",
860  agent->is_active ? "yes" : "no");
861  new_address->active = agent->is_active;
862  new_address->assigned_bw_in = agent->bw_in;
863  new_address->assigned_bw_out = agent->bw_out;
864  }
865  notify |= GNUNET_YES;
866  }
867 
868  if (new_address)
869  {
870  // activity change
871  if (new_address->active != agent->is_active)
872  {
873  new_address->active = agent->is_active;
874  notify |= GNUNET_YES;
875  }
876 
877  // bw change
878  if (agent->bw_in != new_bw_in)
879  {
880  agent->bw_in = new_bw_in;
881  new_address->assigned_bw_in = new_bw_in;
882  notify |= GNUNET_YES;
883  }
884  if (agent->bw_out != new_bw_out)
885  {
886  agent->bw_out = new_bw_out;
887  new_address->assigned_bw_out = new_bw_out;
888  notify |= GNUNET_YES;
889  }
890  }
891 
892  if (notify && agent->is_active && (GNUNET_NO == silent))
893  {
894  if (new_address)
895  {
897  " envi_set_active_suggestion() notify\n");
898  agent->suggestion_issue = GNUNET_YES;
899  agent->suggestion_address = new_address;
900  }
901  else if (agent->address_inuse)
902  {
903  /* disconnect case, no new address */
906  agent->bw_in = 0;
907  agent->bw_out = 0;
908 
909  agent->suggestion_issue = GNUNET_YES;
910  agent->suggestion_address = agent->address_inuse;
911  }
912  }
913  agent->address_inuse = new_address;
914 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
int is_active
Whether the agent is active or not.
int active
Is this the active address for this peer?
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
#define GNUNET_NO
Definition: gnunet_common.h:78
int suggestion_issue
Flag whether a suggestion has to be issued.
uint32_t assigned_bw_in
Inbound bandwidth assigned by solver.
struct GNUNET_PeerIdentity peer
Peer ID.
static int silent
–silent option
Definition: gnunet-qr.c:49
static struct GNUNET_PEERINFO_NotifyContext * notify
Handle to the peerinfo notify service (NULL until we&#39;ve connected to it).
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
uint32_t assigned_bw_out
Outbound bandwidth assigned by solver.
#define GNUNET_YES
Definition: gnunet_common.h:77
#define LOG(kind,...)
struct ATS_Address * suggestion_address
The address which has to be issued.
const char * GNUNET_i2s(const struct GNUNET_PeerIdentity *pid)
Convert a peer identity to a string (for printing debug messages).
Here is the call graph for this function:
Here is the caller graph for this function:

◆ envi_get_state()

static double* envi_get_state ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent 
)
static

Allocates a state vector and fills it with the features present.

Parameters
solverthe solver handle
agentthe agent handle
Returns
pointer to the state vector

Definition at line 924 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, agent_address_get_index(), RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, removetrailingwhitespace::f, GNUNET_malloc, m, RIL_Peer_Agent::m, GAS_RIL_Handle::parameters, RIL_Learning_Parameters::rbf_divisor, ril_get_max_bw(), and state.

Referenced by agent_step().

925 {
926  double *state;
927  double y[2];
928  double x[2];
929  double d[2];
930  double sigma;
931  double f;
932  int m;
933  int i;
934  int k;
935  unsigned long long max_bw;
936 
937  state = GNUNET_malloc (sizeof(double) * agent->m);
938 
939  max_bw = ril_get_max_bw ((struct
940  RIL_Scope *) agent->address_inuse->
941  solver_information);
942 
943  y[0] = (double) agent->bw_out;
944  y[1] = (double) agent->bw_in;
945 
946  m = agent_address_get_index (agent, agent->address_inuse)
947  * (solver->parameters.rbf_divisor + 1) * (solver->parameters.rbf_divisor
948  + 1);
949  for (i = 0; i <= solver->parameters.rbf_divisor; i++)
950  {
951  for (k = 0; k <= solver->parameters.rbf_divisor; k++)
952  {
953  x[0] = (double) i * (double) max_bw
954  / (double) solver->parameters.rbf_divisor;
955  x[1] = (double) k * (double) max_bw
956  / (double) solver->parameters.rbf_divisor;
957  d[0] = x[0] - y[0];
958  d[1] = x[1] - y[1];
959  sigma = (((double) max_bw / ((double) solver->parameters.rbf_divisor
960  + 1)) * 0.5);
961  f = exp (-((d[0] * d[0] + d[1] * d[1]) / (2 * sigma * sigma)));
962  state[m++] = f;
963  }
964  }
965 
966  return state;
967 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
unsigned int m
Number of rows of W / Number of state-vector features.
static struct GNUNET_ARM_MonitorHandle * m
Monitor connection with ARM.
Definition: gnunet-arm.c:104
enum State state
current state of profiling
static unsigned long long ril_get_max_bw(struct RIL_Scope *net)
Calculates the maximum bandwidth an agent can assign in a network scope.
static int agent_address_get_index(struct RIL_Peer_Agent *agent, struct ATS_Address *address)
Get the index of the address in the agent&#39;s list.
struct RIL_Learning_Parameters parameters
Learning parameters.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
unsigned long long rbf_divisor
State space divisor.
#define GNUNET_malloc(size)
Wrapper around malloc.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_get_utility()

static double agent_get_utility ( struct RIL_Peer_Agent agent)
static

Returns the utility value of the connection an agent manages.

Parameters
agentthe agent in question
Returns
the utility value

Definition at line 977 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, GNUNET_ATS_PluginEnvironment::cls, GAS_RIL_Handle::env, RIL_Peer_Agent::envi, GNUNET_ATS_PluginEnvironment::get_preferences, GNUNET_ATS_PREFERENCE_BANDWIDTH, GNUNET_ATS_PREFERENCE_LATENCY, GAS_NormalizationInfo::norm, ATS_Address::norm_delay, RIL_Peer_Agent::peer, RIL_MIN_BW, and RIL_UTILITY_DELAY_MAX.

Referenced by envi_get_reward(), and ril_network_get_social_welfare().

978 {
979  const double *preferences;
980  double delay_atsi;
981  double delay_norm;
982  double pref_match;
983 
984  preferences = agent->envi->env->get_preferences (agent->envi->env->cls,
985  &agent->peer);
986 
987  delay_atsi = agent->address_inuse->norm_delay.norm;
988  delay_norm = RIL_UTILITY_DELAY_MAX * exp (-delay_atsi * 0.00001);
989 
990  pref_match = preferences[GNUNET_ATS_PREFERENCE_LATENCY] * delay_norm;
991  pref_match += preferences[GNUNET_ATS_PREFERENCE_BANDWIDTH]
992  * sqrt ((double) (agent->bw_in / RIL_MIN_BW)
993  * (double) (agent->bw_out / RIL_MIN_BW));
994  return pref_match;
995 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
struct GNUNET_ATS_PluginEnvironment * env
The solver-plugin environment of the solver-plugin API.
#define RIL_UTILITY_DELAY_MAX
GAS_get_preferences get_preferences
ATS addresses function to obtain preference values.
struct GNUNET_PeerIdentity peer
Peer ID.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
Change the peer&#39;s latency value to the given amount.
Change the peer&#39;s bandwidth value (value per byte of bandwidth in the goal function) to the given amo...
struct GAS_NormalizationInfo norm_delay
Normalized delay information for this address.
void * cls
Closure to pass to all callbacks in this struct.
double norm
Normalized values from queue to a range of values [1.0...2.0].
struct GAS_RIL_Handle * envi
Environment handle.
#define RIL_MIN_BW
Here is the caller graph for this function:

◆ ril_network_get_social_welfare()

static double ril_network_get_social_welfare ( struct GAS_RIL_Handle solver,
struct RIL_Scope scope 
)
static

Calculates the social welfare within a network scope according to what social welfare measure is set in the configuration.

Parameters
solverthe solver handle
scopethe network scope in question
Returns
the social welfare value

Definition at line 1006 of file plugin_ats_ril.c.

References RIL_Scope::active_agent_count, RIL_Peer_Agent::address_inuse, agent_get_utility(), GAS_RIL_Handle::agents_head, GNUNET_assert, GNUNET_MIN, GNUNET_NO, RIL_Peer_Agent::is_active, RIL_Peer_Agent::next, GAS_RIL_Handle::parameters, result, RIL_WELFARE_EGALITARIAN, RIL_WELFARE_NASH, RIL_Learning_Parameters::social_welfare, and ATS_Address::solver_information.

Referenced by ril_networks_update_state().

1008 {
1009  struct RIL_Peer_Agent *cur;
1010  double result;
1011 
1012  switch (solver->parameters.social_welfare)
1013  {
1015  result = DBL_MAX;
1016  for (cur = solver->agents_head; NULL != cur; cur = cur->next)
1017  {
1018  if (cur->is_active && cur->address_inuse &&
1019  (cur->address_inuse->solver_information == scope))
1020  {
1021  result = GNUNET_MIN (result, agent_get_utility (cur));
1022  }
1023  }
1024  return result;
1025 
1026  case RIL_WELFARE_NASH:
1027  result = 0;
1028  for (cur = solver->agents_head; NULL != cur; cur = cur->next)
1029  {
1030  if (cur->is_active && cur->address_inuse &&
1031  (cur->address_inuse->solver_information == scope))
1032  {
1033  result *= pow (agent_get_utility (cur), 1.0
1034  / (double) scope->active_agent_count);
1035  }
1036  }
1037  return result;
1038  }
1040  return 1;
1041 }
struct RIL_Peer_Agent * agents_head
List of active peer-agents.
int is_active
Whether the agent is active or not.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
#define GNUNET_NO
Definition: gnunet_common.h:78
enum RIL_Welfare social_welfare
Which measure of social welfare should be used.
#define GNUNET_MIN(a, b)
Definition: gnunet_common.h:80
static int result
Global testing status.
struct RIL_Learning_Parameters parameters
Learning parameters.
void * solver_information
Solver-specific information for this address.
struct ATS_Address * address_inuse
Address in use.
unsigned int active_agent_count
Number of active agents in scope.
static double agent_get_utility(struct RIL_Peer_Agent *agent)
Returns the utility value of the connection an agent manages.
struct RIL_Peer_Agent * next
Next agent in solver&#39;s linked list.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ envi_get_penalty()

static double envi_get_penalty ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent 
)
static

Definition at line 1044 of file plugin_ats_ril.c.

References RIL_Peer_Agent::a_old, RIL_Peer_Agent::address_inuse, RIL_Scope::bw_in_available, RIL_Scope::bw_in_utilized, RIL_Scope::bw_out_available, RIL_Scope::bw_out_utilized, RIL_ACTION_BW_IN_INC, RIL_ACTION_BW_OUT_INC, RIL_MIN_BW, and ATS_Address::solver_information.

Referenced by envi_get_reward().

1045 {
1046  struct RIL_Scope *net;
1047  unsigned long long over_max;
1048  unsigned long long over_in = 0;
1049  unsigned long long over_out = 0;
1050 
1051  net = agent->address_inuse->solver_information;
1052 
1053  if (net->bw_in_utilized > net->bw_in_available)
1054  {
1055  over_in = net->bw_in_utilized - net->bw_in_available;
1056  if (RIL_ACTION_BW_IN_INC == agent->a_old)
1057  {
1058  /* increase quadratically */
1059  over_in *= over_in;
1060  }
1061  }
1062  if (net->bw_out_utilized > net->bw_out_available)
1063  {
1064  over_out = net->bw_out_utilized - net->bw_out_available;
1065  if (RIL_ACTION_BW_OUT_INC == agent->a_old)
1066  {
1067  /* increase quadratically */
1068  over_out *= over_out;
1069  }
1070  }
1071  over_max = (over_in + over_out) / (RIL_MIN_BW * RIL_MIN_BW);
1072 
1073  return -1.0 * (double) over_max;
1074 }
int a_old
Last chosen action.
unsigned long long bw_out_utilized
Bandwidth outbound actually utilized in the network.
uint32_t bw_in_utilized
Bandwidth inbound actually utilized in the network.
void * solver_information
Solver-specific information for this address.
struct ATS_Address * address_inuse
Address in use.
uint32_t bw_out_available
Total available outbound bandwidth.
uint32_t bw_in_available
Total available inbound bandwidth.
#define RIL_MIN_BW
Here is the caller graph for this function:

◆ envi_get_reward()

static double envi_get_reward ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent 
)
static

Gets the reward for the last performed step, which is calculated in equal parts from the local (the peer specific) and the global (for all peers identical) reward.

Parameters
solverthe solver handle
agentthe agent handle
Returns
the reward

Definition at line 1086 of file plugin_ats_ril.c.

References RIL_Peer_Agent::a_old, RIL_Peer_Agent::address_inuse, agent_get_utility(), delta, envi_get_penalty(), RIL_Peer_Agent::nop_bonus, RIL_Peer_Agent::objective_old, RIL_ACTION_NOTHING, RIL_NOP_DECAY, RIL_Scope::social_welfare, and ATS_Address::solver_information.

Referenced by agent_step().

1087 {
1088  struct RIL_Scope *net;
1089  double objective;
1090  double delta;
1091  double steady;
1092  double penalty;
1093  double reward;
1094 
1095  net = agent->address_inuse->solver_information;
1096 
1097  penalty = envi_get_penalty (solver, agent);
1098  objective = (agent_get_utility (agent) + net->social_welfare) / 2;
1099  delta = objective - agent->objective_old;
1100  agent->objective_old = objective;
1101 
1102  if ((delta != 0)&&(penalty == 0))
1103  {
1104  agent->nop_bonus = delta * RIL_NOP_DECAY;
1105  }
1106  else
1107  {
1108  agent->nop_bonus *= RIL_NOP_DECAY;
1109  }
1110 
1111  steady = (RIL_ACTION_NOTHING == agent->a_old) ? agent->nop_bonus : 0;
1112 
1113  reward = delta + steady;
1114  return reward + penalty;
1115 }
static struct GNUNET_TIME_Relative delta
Definition: speedup.c:35
int a_old
Last chosen action.
double social_welfare
The social welfare achieved in the scope.
void * solver_information
Solver-specific information for this address.
struct ATS_Address * address_inuse
Address in use.
#define RIL_NOP_DECAY
static double agent_get_utility(struct RIL_Peer_Agent *agent)
Returns the utility value of the connection an agent manages.
static double envi_get_penalty(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent)
double nop_bonus
NOP bonus.
double objective_old
The agent&#39;s last objective value.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ envi_action_bw_double()

static void envi_action_bw_double ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent,
int  direction_in 
)
static

Doubles the bandwidth for the active address.

Parameters
solversolver handle
agentagent handle
direction_inif GNUNET_YES, change inbound bandwidth, otherwise the outbound bandwidth

Definition at line 1125 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, envi_set_active_suggestion(), GNUNET_NO, and ril_get_max_bw().

Referenced by envi_do_action().

1128 {
1129  unsigned long long new_bw;
1130  unsigned long long max_bw;
1131 
1132  max_bw = ril_get_max_bw ((struct
1133  RIL_Scope *) agent->address_inuse->
1134  solver_information);
1135 
1136  if (direction_in)
1137  {
1138  new_bw = agent->bw_in * 2;
1139  if ((new_bw < agent->bw_in) ||(new_bw > max_bw) )
1140  new_bw = max_bw;
1141  envi_set_active_suggestion (solver, agent, agent->address_inuse, new_bw,
1142  agent->bw_out, GNUNET_NO);
1143  }
1144  else
1145  {
1146  new_bw = agent->bw_out * 2;
1147  if ((new_bw < agent->bw_out) ||(new_bw > max_bw) )
1148  new_bw = max_bw;
1149  envi_set_active_suggestion (solver, agent, agent->address_inuse,
1150  agent->bw_in,
1151  new_bw, GNUNET_NO);
1152  }
1153 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
#define GNUNET_NO
Definition: gnunet_common.h:78
static void envi_set_active_suggestion(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, struct ATS_Address *new_address, unsigned long long new_bw_in, unsigned long long new_bw_out, int silent)
Changes the active assignment suggestion of the handler and invokes the bw_changed callback to notify...
static unsigned long long ril_get_max_bw(struct RIL_Scope *net)
Calculates the maximum bandwidth an agent can assign in a network scope.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ envi_action_bw_halven()

static void envi_action_bw_halven ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent,
int  direction_in 
)
static

Cuts the bandwidth for the active address in half.

The least amount of bandwidth suggested, is the minimum bandwidth for a peer, in order to not invoke a disconnect.

Parameters
solversolver handle
agentagent handle
direction_inif GNUNET_YES, change inbound bandwidth, otherwise change the outbound bandwidth

Definition at line 1165 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, envi_set_active_suggestion(), and GNUNET_NO.

Referenced by envi_do_action().

1168 {
1169  unsigned long long new_bw;
1170 
1171  if (direction_in)
1172  {
1173  new_bw = agent->bw_in / 2;
1174  if ((new_bw <= 0) ||(new_bw > agent->bw_in) )
1175  new_bw = 0;
1176  envi_set_active_suggestion (solver, agent, agent->address_inuse, new_bw,
1177  agent->bw_out,
1178  GNUNET_NO);
1179  }
1180  else
1181  {
1182  new_bw = agent->bw_out / 2;
1183  if ((new_bw <= 0) ||(new_bw > agent->bw_out) )
1184  new_bw = 0;
1185  envi_set_active_suggestion (solver, agent, agent->address_inuse,
1186  agent->bw_in, new_bw,
1187  GNUNET_NO);
1188  }
1189 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
#define GNUNET_NO
Definition: gnunet_common.h:78
static void envi_set_active_suggestion(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, struct ATS_Address *new_address, unsigned long long new_bw_in, unsigned long long new_bw_out, int silent)
Changes the active assignment suggestion of the handler and invokes the bw_changed callback to notify...
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ envi_action_bw_inc()

static void envi_action_bw_inc ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent,
int  direction_in 
)
static

Increases the bandwidth by 5 times the minimum bandwidth for the active address.

Parameters
solversolver handle
agentagent handle
direction_inif GNUNET_YES, change inbound bandwidth, otherwise change the outbound bandwidth

Definition at line 1200 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, envi_set_active_suggestion(), GNUNET_NO, ril_get_max_bw(), RIL_INC_DEC_STEP_SIZE, and RIL_MIN_BW.

Referenced by envi_do_action().

1202 {
1203  unsigned long long new_bw;
1204  unsigned long long max_bw;
1205 
1206  max_bw = ril_get_max_bw ((struct
1207  RIL_Scope *) agent->address_inuse->
1208  solver_information);
1209 
1210  if (direction_in)
1211  {
1212  new_bw = agent->bw_in + (RIL_INC_DEC_STEP_SIZE * RIL_MIN_BW);
1213  if ((new_bw < agent->bw_in) ||(new_bw > max_bw) )
1214  new_bw = max_bw;
1215  envi_set_active_suggestion (solver, agent, agent->address_inuse, new_bw,
1216  agent->bw_out, GNUNET_NO);
1217  }
1218  else
1219  {
1220  new_bw = agent->bw_out + (RIL_INC_DEC_STEP_SIZE * RIL_MIN_BW);
1221  if ((new_bw < agent->bw_out) ||(new_bw > max_bw) )
1222  new_bw = max_bw;
1223  envi_set_active_suggestion (solver, agent, agent->address_inuse,
1224  agent->bw_in,
1225  new_bw, GNUNET_NO);
1226  }
1227 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
#define GNUNET_NO
Definition: gnunet_common.h:78
static void envi_set_active_suggestion(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, struct ATS_Address *new_address, unsigned long long new_bw_in, unsigned long long new_bw_out, int silent)
Changes the active assignment suggestion of the handler and invokes the bw_changed callback to notify...
static unsigned long long ril_get_max_bw(struct RIL_Scope *net)
Calculates the maximum bandwidth an agent can assign in a network scope.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
#define RIL_INC_DEC_STEP_SIZE
#define RIL_MIN_BW
Here is the call graph for this function:
Here is the caller graph for this function:

◆ envi_action_bw_dec()

static void envi_action_bw_dec ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent,
int  direction_in 
)
static

Decreases the bandwidth by 5 times the minimum bandwidth for the active address.

The least amount of bandwidth suggested, is the minimum bandwidth for a peer, in order to not invoke a disconnect.

Parameters
solversolver handle
agentagent handle
direction_inif GNUNET_YES, change inbound bandwidth, otherwise change the outbound bandwidth

Definition at line 1239 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, envi_set_active_suggestion(), GNUNET_NO, RIL_INC_DEC_STEP_SIZE, and RIL_MIN_BW.

Referenced by envi_do_action().

1241 {
1242  unsigned long long new_bw;
1243 
1244  if (direction_in)
1245  {
1246  new_bw = agent->bw_in - (RIL_INC_DEC_STEP_SIZE * RIL_MIN_BW);
1247  if ((new_bw <= 0) ||(new_bw > agent->bw_in) )
1248  new_bw = 0;
1249  envi_set_active_suggestion (solver, agent, agent->address_inuse, new_bw,
1250  agent->bw_out,
1251  GNUNET_NO);
1252  }
1253  else
1254  {
1255  new_bw = agent->bw_out - (RIL_INC_DEC_STEP_SIZE * RIL_MIN_BW);
1256  if ((new_bw <= 0) ||(new_bw > agent->bw_out) )
1257  new_bw = 0;
1258  envi_set_active_suggestion (solver, agent, agent->address_inuse,
1259  agent->bw_in, new_bw,
1260  GNUNET_NO);
1261  }
1262 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
#define GNUNET_NO
Definition: gnunet_common.h:78
static void envi_set_active_suggestion(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, struct ATS_Address *new_address, unsigned long long new_bw_in, unsigned long long new_bw_out, int silent)
Changes the active assignment suggestion of the handler and invokes the bw_changed callback to notify...
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
#define RIL_INC_DEC_STEP_SIZE
#define RIL_MIN_BW
Here is the call graph for this function:
Here is the caller graph for this function:

◆ envi_action_address_switch()

static void envi_action_address_switch ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent,
unsigned int  address_index 
)
static

Switches to the address given by its index.

Parameters
solversolver handle
agentagent handle
address_indexindex of the address as it is saved in the agent's list, starting with zero

Definition at line 1272 of file plugin_ats_ril.c.

References RIL_Address_Wrapped::address_naked, RIL_Peer_Agent::addresses_head, RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, envi_set_active_suggestion(), GNUNET_assert, GNUNET_NO, and RIL_Address_Wrapped::next.

Referenced by envi_do_action().

1275 {
1276  struct RIL_Address_Wrapped *cur;
1277  int i = 0;
1278 
1279  // cur = agent_address_get_wrapped(agent, agent->address_inuse);
1280 
1281  for (cur = agent->addresses_head; NULL != cur; cur = cur->next)
1282  {
1283  if (i == address_index)
1284  {
1285  envi_set_active_suggestion (solver, agent, cur->address_naked,
1286  agent->bw_in, agent->bw_out,
1287  GNUNET_NO);
1288  return;
1289  }
1290 
1291  i++;
1292  }
1293 
1294  // no address with address_index exists, in this case this action should not be callable
1296 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
struct RIL_Address_Wrapped * addresses_head
Head of addresses DLL.
#define GNUNET_NO
Definition: gnunet_common.h:78
static void envi_set_active_suggestion(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, struct ATS_Address *new_address, unsigned long long new_bw_in, unsigned long long new_bw_out, int silent)
Changes the active assignment suggestion of the handler and invokes the bw_changed callback to notify...
struct ATS_Address * address_naked
The address.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct RIL_Address_Wrapped * next
Next in DLL.
Wrapper for addresses to store them in agent&#39;s linked list.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ envi_do_action()

static void envi_do_action ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent,
int  action 
)
static

Puts the action into effect by calling the according function.

Parameters
solverthe solver handle
agentthe action handle
actionthe action to perform by the solver

Definition at line 1306 of file plugin_ats_ril.c.

References RIL_Address_Wrapped::address_naked, RIL_Peer_Agent::addresses_tail, agent_address_get_index(), envi_action_address_switch(), envi_action_bw_dec(), envi_action_bw_double(), envi_action_bw_halven(), envi_action_bw_inc(), GNUNET_assert, GNUNET_NO, GNUNET_YES, RIL_ACTION_BW_IN_DBL, RIL_ACTION_BW_IN_DEC, RIL_ACTION_BW_IN_HLV, RIL_ACTION_BW_IN_INC, RIL_ACTION_BW_OUT_DBL, RIL_ACTION_BW_OUT_DEC, RIL_ACTION_BW_OUT_HLV, RIL_ACTION_BW_OUT_INC, RIL_ACTION_NOTHING, and RIL_ACTION_TYPE_NUM.

Referenced by agent_step().

1308 {
1309  int address_index;
1310 
1311  switch (action)
1312  {
1313  case RIL_ACTION_NOTHING:
1314  break;
1315 
1316  case RIL_ACTION_BW_IN_DBL:
1317  envi_action_bw_double (solver, agent, GNUNET_YES);
1318  break;
1319 
1320  case RIL_ACTION_BW_IN_HLV:
1321  envi_action_bw_halven (solver, agent, GNUNET_YES);
1322  break;
1323 
1324  case RIL_ACTION_BW_IN_INC:
1325  envi_action_bw_inc (solver, agent, GNUNET_YES);
1326  break;
1327 
1328  case RIL_ACTION_BW_IN_DEC:
1329  envi_action_bw_dec (solver, agent, GNUNET_YES);
1330  break;
1331 
1332  case RIL_ACTION_BW_OUT_DBL:
1333  envi_action_bw_double (solver, agent, GNUNET_NO);
1334  break;
1335 
1336  case RIL_ACTION_BW_OUT_HLV:
1337  envi_action_bw_halven (solver, agent, GNUNET_NO);
1338  break;
1339 
1340  case RIL_ACTION_BW_OUT_INC:
1341  envi_action_bw_inc (solver, agent, GNUNET_NO);
1342  break;
1343 
1344  case RIL_ACTION_BW_OUT_DEC:
1345  envi_action_bw_dec (solver, agent, GNUNET_NO);
1346  break;
1347 
1348  default:
1349  if ((action >= RIL_ACTION_TYPE_NUM) && (action < agent->n)) // switch address action
1350  {
1351  address_index = action - RIL_ACTION_TYPE_NUM;
1352 
1353  GNUNET_assert (address_index >= 0);
1354  GNUNET_assert (
1355  address_index <= agent_address_get_index (agent,
1356  agent->addresses_tail->
1357  address_naked));
1358 
1359  envi_action_address_switch (solver, agent, address_index);
1360  break;
1361  }
1362  // error - action does not exist
1364  }
1365 }
static void envi_action_bw_halven(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int direction_in)
Cuts the bandwidth for the active address in half.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
static void envi_action_bw_dec(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int direction_in)
Decreases the bandwidth by 5 times the minimum bandwidth for the active address.
#define GNUNET_NO
Definition: gnunet_common.h:78
struct ATS_Address * address_naked
The address.
static void envi_action_address_switch(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, unsigned int address_index)
Switches to the address given by its index.
static int agent_address_get_index(struct RIL_Peer_Agent *agent, struct ATS_Address *address)
Get the index of the address in the agent&#39;s list.
static void envi_action_bw_double(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int direction_in)
Doubles the bandwidth for the active address.
#define GNUNET_YES
Definition: gnunet_common.h:77
struct RIL_Address_Wrapped * addresses_tail
Tail of addresses DLL.
static void envi_action_bw_inc(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int direction_in)
Increases the bandwidth by 5 times the minimum bandwidth for the active address.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_select_egreedy()

static int agent_select_egreedy ( struct RIL_Peer_Agent agent,
double *  state 
)
static

Selects the next action using the e-greedy strategy.

I.e. with a probability of (1-e) the action with the maximum expected return will be chosen (=> exploitation) and with probability (e) a random action will be chosen. In case the Q-learning rule is set, the function also resets the eligibility traces in the exploration case (after Watkin's Q-learning).

Parameters
agentthe agent selecting an action
statethe current state-feature vector
Returns
the action index

Definition at line 1379 of file plugin_ats_ril.c.

References agent_get_action_max(), agent_get_action_random(), RIL_Learning_Parameters::algorithm, RIL_Peer_Agent::eligibility_reset, RIL_Peer_Agent::envi, RIL_Learning_Parameters::epsilon, RIL_Learning_Parameters::epsilon_decay, GNUNET_CRYPTO_QUALITY_WEAK, GNUNET_CRYPTO_random_u32(), GNUNET_YES, GAS_RIL_Handle::parameters, and RIL_ALGO_Q.

Referenced by agent_select_action().

1380 {
1381  int action;
1383  UINT32_MAX)
1384  / (double) UINT32_MAX;
1385 
1386  if (r < agent->envi->parameters.epsilon) // explore
1387  {
1388  action = agent_get_action_random (agent);
1389  if (RIL_ALGO_Q == agent->envi->parameters.algorithm)
1390  {
1391  agent->eligibility_reset = GNUNET_YES;
1392  }
1393  agent->envi->parameters.epsilon *= agent->envi->parameters.epsilon_decay;
1394  return action;
1395  }
1396  else // exploit
1397  {
1398  action = agent_get_action_max (agent, state);
1399  return action;
1400  }
1401 }
uint32_t GNUNET_CRYPTO_random_u32(enum GNUNET_CRYPTO_Quality mode, uint32_t i)
Produce a random value.
static int agent_get_action_max(struct RIL_Peer_Agent *agent, double *state)
Gets the action, with the maximal estimated Q-value (i.e.
int eligibility_reset
Whether to reset the eligibility traces to 0 after a Q-exploration step.
double epsilon
Ratio, with what probability an agent should explore in the e-greed policy.
static int agent_get_action_random(struct RIL_Peer_Agent *agent)
Chooses a random action from the set of possible ones.
enum State state
current state of profiling
enum RIL_Algorithm algorithm
The TD-algorithm to use.
struct RIL_Learning_Parameters parameters
Learning parameters.
double epsilon_decay
Decay factor of the explore ratio.
#define GNUNET_YES
Definition: gnunet_common.h:77
struct GAS_RIL_Handle * envi
Environment handle.
No good quality of the operation is needed (i.e., random numbers can be pseudo-random).
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_select_softmax()

static int agent_select_softmax ( struct RIL_Peer_Agent agent,
double *  state 
)
static

Selects the next action with a probability corresponding to its value.

The probability is calculated using a Boltzmann distribution with a temperature value. The higher the temperature, the more are the action selection probabilities the same. With a temperature of 0, the selection is greedy, i.e. always the action with the highest value is chosen.

Parameters
agent
state
Returns

Definition at line 1414 of file plugin_ats_ril.c.

References agent_action_is_possible(), agent_get_action_max(), agent_q(), RIL_Learning_Parameters::algorithm, RIL_Peer_Agent::eligibility_reset, RIL_Peer_Agent::envi, GNUNET_assert, GNUNET_CRYPTO_QUALITY_WEAK, GNUNET_CRYPTO_random_u32(), GNUNET_NO, GNUNET_YES, RIL_Peer_Agent::n, p, GAS_RIL_Handle::parameters, RIL_ACTION_INVALID, RIL_ALGO_Q, RIL_Learning_Parameters::temperature, and RIL_Learning_Parameters::temperature_decay.

Referenced by agent_select_action().

1415 {
1416  int i;
1417  int a_max;
1418  double eqt[agent->n];
1419  double p[agent->n];
1420  double sum = 0;
1421  double r;
1422 
1423  a_max = agent_get_action_max (agent, state);
1424 
1425  for (i = 0; i < agent->n; i++)
1426  {
1427  if (agent_action_is_possible (agent, i))
1428  {
1429  eqt[i] = exp (agent_q (agent, state, i)
1430  / agent->envi->parameters.temperature);
1431  if (isinf (eqt[i]))
1432  eqt[i] = isinf (eqt[i]) * UINT32_MAX;
1433  sum += eqt[i];
1434  }
1435  }
1436  for (i = 0; i < agent->n; i++)
1437  {
1438  if (agent_action_is_possible (agent, i))
1439  {
1440  p[i] = eqt[i] / sum;
1441  }
1442  else
1443  {
1444  p[i] = 0;
1445  }
1446  }
1448  UINT32_MAX) / (double) UINT32_MAX;
1449  sum = 0;
1450  for (i = 0; i < agent->n; i++)
1451  {
1452  if (sum + p[i] > r)
1453  {
1454  if (i != a_max)
1455  {
1456  if (RIL_ALGO_Q == agent->envi->parameters.algorithm)
1457  agent->eligibility_reset = GNUNET_YES;
1458  agent->envi->parameters.temperature *=
1460  }
1461  return i;
1462  }
1463  sum += p[i];
1464  }
1466  return RIL_ACTION_INVALID;
1467 }
uint32_t GNUNET_CRYPTO_random_u32(enum GNUNET_CRYPTO_Quality mode, uint32_t i)
Produce a random value.
static double agent_q(struct RIL_Peer_Agent *agent, const double *state, int action)
Estimate the current action-value for state s and action a.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
static int agent_get_action_max(struct RIL_Peer_Agent *agent, double *state)
Gets the action, with the maximal estimated Q-value (i.e.
int eligibility_reset
Whether to reset the eligibility traces to 0 after a Q-exploration step.
double temperature_decay
Decay factor of the temperature value.
#define GNUNET_NO
Definition: gnunet_common.h:78
double temperature
Softmax action-selection temperature.
enum State state
current state of profiling
static struct GNUNET_OS_Process * p
Helper process we started.
Definition: gnunet-qr.c:59
enum RIL_Algorithm algorithm
The TD-algorithm to use.
struct RIL_Learning_Parameters parameters
Learning parameters.
unsigned int n
Number of columns of W / Number of actions.
#define RIL_ACTION_INVALID
#define GNUNET_YES
Definition: gnunet_common.h:77
static int agent_action_is_possible(struct RIL_Peer_Agent *agent, int action)
struct GAS_RIL_Handle * envi
Environment handle.
No good quality of the operation is needed (i.e., random numbers can be pseudo-random).
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_select_action()

static int agent_select_action ( struct RIL_Peer_Agent agent,
double *  state 
)
static

Select the next action of an agent either according to the e-greedy strategy or the softmax strategy.

Parameters
agentthe agent in question
statethe current state-feature vector
Returns
the action index

Definition at line 1478 of file plugin_ats_ril.c.

References agent_select_egreedy(), agent_select_softmax(), RIL_Peer_Agent::envi, GAS_RIL_Handle::parameters, RIL_SELECT_EGREEDY, and RIL_Learning_Parameters::select.

Referenced by agent_step().

1479 {
1480  if (agent->envi->parameters.select == RIL_SELECT_EGREEDY)
1481  {
1482  return agent_select_egreedy (agent, state);
1483  }
1484  else
1485  {
1486  return agent_select_softmax (agent, state);
1487  }
1488 }
static int agent_select_softmax(struct RIL_Peer_Agent *agent, double *state)
Selects the next action with a probability corresponding to its value.
enum State state
current state of profiling
enum RIL_Select select
Action selection strategy;.
static int agent_select_egreedy(struct RIL_Peer_Agent *agent, double *state)
Selects the next action using the e-greedy strategy.
struct RIL_Learning_Parameters parameters
Learning parameters.
struct GAS_RIL_Handle * envi
Environment handle.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_step()

static void agent_step ( struct RIL_Peer_Agent agent)
static

Performs one step of the Markov Decision Process.

Other than in the literature the step starts after having done the last action a_old. It observes the new state s_next and the reward received. Then the coefficient update is done according to the SARSA or Q-learning method. The next action is put into effect.

Parameters
agentthe agent performing the step

Definition at line 1499 of file plugin_ats_ril.c.

References RIL_Peer_Agent::a_old, agent_get_action_max(), agent_modify_eligibility(), agent_select_action(), agent_update(), RIL_Learning_Parameters::algorithm, RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, RIL_Peer_Agent::eligibility_reset, RIL_Learning_Parameters::eligibility_trace_mode, RIL_Peer_Agent::envi, envi_do_action(), envi_get_reward(), envi_get_state(), GNUNET_assert, GNUNET_ERROR_TYPE_DEBUG, GNUNET_free, GNUNET_i2s(), GNUNET_NO, LOG, GAS_RIL_Handle::parameters, RIL_Peer_Agent::peer, RIL_ACTION_INVALID, RIL_ALGO_Q, RIL_ALGO_SARSA, RIL_E_DECAY, RIL_E_ZERO, ril_step(), RIL_Peer_Agent::s_old, and RIL_Peer_Agent::step_count.

Referenced by ril_step().

1500 {
1501  int a_next = RIL_ACTION_INVALID;
1502  int a_max;
1503  double *s_next;
1504  double reward;
1505 
1506  LOG (GNUNET_ERROR_TYPE_DEBUG, " agent_step() Peer '%s', algorithm %s\n",
1507  GNUNET_i2s (&agent->peer),
1508  agent->envi->parameters.algorithm ? "Q" : "SARSA");
1509 
1510  s_next = envi_get_state (agent->envi, agent);
1511  reward = envi_get_reward (agent->envi, agent);
1512 
1513  if (agent->eligibility_reset)
1514  {
1515  agent_modify_eligibility (agent, RIL_E_ZERO, NULL, -1);
1516  agent->eligibility_reset = GNUNET_NO;
1517  }
1518  else
1519  {
1520  agent_modify_eligibility (agent, RIL_E_DECAY, NULL, -1);
1521  }
1522  if (RIL_ACTION_INVALID != agent->a_old)
1523  {
1524  agent_modify_eligibility (agent,
1526  agent->s_old, agent->a_old);
1527  }
1528 
1529  switch (agent->envi->parameters.algorithm)
1530  {
1531  case RIL_ALGO_SARSA:
1532  a_next = agent_select_action (agent, s_next);
1533  if (RIL_ACTION_INVALID != agent->a_old)
1534  {
1535  // updates weights with selected action (on-policy), if not first step
1536  agent_update (agent, reward, s_next, a_next);
1537  }
1538  break;
1539 
1540  case RIL_ALGO_Q:
1541  a_max = agent_get_action_max (agent, s_next);
1542  if (RIL_ACTION_INVALID != agent->a_old)
1543  {
1544  // updates weights with best action, disregarding actually selected action (off-policy), if not first step
1545  agent_update (agent, reward, s_next, a_max);
1546  }
1547  a_next = agent_select_action (agent, s_next);
1548  break;
1549  }
1550 
1551  GNUNET_assert (RIL_ACTION_INVALID != a_next);
1552 
1554  "step() Step# %llu R: %f IN %llu OUT %llu A: %d\n",
1555  agent->step_count,
1556  reward,
1557  agent->bw_in / 1024,
1558  agent->bw_out / 1024,
1559  a_next);
1560 
1561  envi_do_action (agent->envi, agent, a_next);
1562 
1563  GNUNET_free (agent->s_old);
1564  agent->s_old = s_next;
1565  agent->a_old = a_next;
1566 
1567  agent->step_count += 1;
1568 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
static int agent_get_action_max(struct RIL_Peer_Agent *agent, double *state)
Gets the action, with the maximal estimated Q-value (i.e.
int eligibility_reset
Whether to reset the eligibility traces to 0 after a Q-exploration step.
int a_old
Last chosen action.
#define GNUNET_NO
Definition: gnunet_common.h:78
static double * envi_get_state(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent)
Allocates a state vector and fills it with the features present.
struct GNUNET_PeerIdentity peer
Peer ID.
static void envi_do_action(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int action)
Puts the action into effect by calling the according function.
enum RIL_Algorithm algorithm
The TD-algorithm to use.
struct RIL_Learning_Parameters parameters
Learning parameters.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
#define RIL_ACTION_INVALID
static void agent_update(struct RIL_Peer_Agent *agent, double reward, double *s_next, int a_prime)
Updates the weights (i.e.
static double envi_get_reward(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent)
Gets the reward for the last performed step, which is calculated in equal parts from the local (the p...
static int agent_select_action(struct RIL_Peer_Agent *agent, double *state)
Select the next action of an agent either according to the e-greedy strategy or the softmax strategy...
double * s_old
Last perceived state feature vector.
#define LOG(kind,...)
static void agent_modify_eligibility(struct RIL_Peer_Agent *agent, enum RIL_E_Modification mod, double *feature, int action)
Changes the eligibility trace vector e in various manners: RIL_E_ACCUMULATE - adds feature to each co...
const char * GNUNET_i2s(const struct GNUNET_PeerIdentity *pid)
Convert a peer identity to a string (for printing debug messages).
struct GAS_RIL_Handle * envi
Environment handle.
#define GNUNET_free(ptr)
Wrapper around free.
unsigned long long step_count
Number of performed time-steps.
enum RIL_E_Modification eligibility_trace_mode
Whether to accumulate or replace eligibility traces.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ ril_step()

static void ril_step ( struct GAS_RIL_Handle solver)
static

Prototype of the ril_step() procedure.

Triggers one step per agent.

Parameters
solverthe solver handle
solver

Definition at line 1958 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, agent_step(), GAS_RIL_Handle::agents_head, GNUNET_ATS_PluginEnvironment::bandwidth_changed_cb, GAS_RIL_Handle::bulk_changes, GAS_RIL_Handle::bulk_lock, GNUNET_ATS_PluginEnvironment::cls, GAS_RIL_Handle::env, GAS_OP_SOLVE_START, GAS_OP_SOLVE_STOP, GAS_OP_SOLVE_UPDATE_NOTIFICATION_START, GAS_OP_SOLVE_UPDATE_NOTIFICATION_STOP, GAS_STAT_SUCCESS, GNUNET_ERROR_TYPE_DEBUG, GNUNET_NO, GNUNET_TIME_absolute_get(), GNUNET_YES, RIL_Peer_Agent::is_active, LOG, RIL_Peer_Agent::next, ril_calculate_discount(), ril_inform(), ril_networks_update_state(), ril_step_schedule_next(), ril_try_unblock_agent(), GAS_RIL_Handle::step_count, GAS_RIL_Handle::step_time_last, RIL_Peer_Agent::suggestion_address, and RIL_Peer_Agent::suggestion_issue.

Referenced by agent_step(), GAS_ril_address_add(), GAS_ril_address_change_preference(), GAS_ril_address_delete(), GAS_ril_address_property_changed(), GAS_ril_bulk_stop(), GAS_ril_stop_get_preferred_address(), and ril_step_scheduler_task().

1959 {
1960  struct RIL_Peer_Agent *cur;
1961 
1962  if (GNUNET_YES == solver->bulk_lock)
1963  {
1964  solver->bulk_changes++;
1965  return;
1966  }
1967 
1969 
1970  LOG (GNUNET_ERROR_TYPE_DEBUG, " RIL step number %d\n", solver->step_count);
1971 
1972  if (0 == solver->step_count)
1973  {
1975  }
1976 
1977  ril_calculate_discount (solver);
1978  ril_networks_update_state (solver);
1979 
1980  // trigger one step per active, unblocked agent
1981  for (cur = solver->agents_head; NULL != cur; cur = cur->next)
1982  {
1983  if (cur->is_active)
1984  {
1985  if (NULL == cur->address_inuse)
1986  {
1987  ril_try_unblock_agent (solver, cur, GNUNET_NO);
1988  }
1989  if (cur->address_inuse)
1990  {
1991  agent_step (cur);
1992  }
1993  }
1994  }
1995 
1996  ril_networks_update_state (solver);
1997 
1998  solver->step_count++;
1999  ril_step_schedule_next (solver);
2000 
2002 
2004  for (cur = solver->agents_head; NULL != cur; cur = cur->next)
2005  {
2006  if (cur->suggestion_issue)
2007  {
2008  solver->env->bandwidth_changed_cb (solver->env->cls,
2009  cur->suggestion_address);
2010  cur->suggestion_issue = GNUNET_NO;
2011  }
2012  }
2014 }
After the problem was finished, start notifications about changes to addresses.
struct RIL_Peer_Agent * agents_head
List of active peer-agents.
int is_active
Whether the agent is active or not.
struct GNUNET_ATS_PluginEnvironment * env
The solver-plugin environment of the solver-plugin API.
int bulk_changes
Number of changes during a lock.
static void ril_inform(struct GAS_RIL_Handle *solver, enum GAS_Solver_Operation op, enum GAS_Solver_Status stat)
Informs the environment about the status of the solver.
#define GNUNET_NO
Definition: gnunet_common.h:78
int suggestion_issue
Flag whether a suggestion has to be issued.
static void ril_try_unblock_agent(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int silent)
Unblocks an agent for which a connection request is there, that could not be satisfied.
unsigned long long step_count
Number of performed steps.
struct GNUNET_TIME_Absolute step_time_last
Timestamp for the last time-step.
A solution iteration has been started.
static void ril_step_schedule_next(struct GAS_RIL_Handle *solver)
Schedules the next global step in an adaptive way.
struct GNUNET_TIME_Absolute GNUNET_TIME_absolute_get(void)
Get the current time.
Definition: time.c:118
static void ril_calculate_discount(struct GAS_RIL_Handle *solver)
Determines how much the reward needs to be discounted depending on the amount of time, which has passed since the last time-step.
struct ATS_Address * address_inuse
Address in use.
A solution iteration has been finished.
static void agent_step(struct RIL_Peer_Agent *agent)
Performs one step of the Markov Decision Process.
static void ril_networks_update_state(struct GAS_RIL_Handle *solver)
Retrieves the state of the network scope, so that its attributes are up-to- date. ...
void * cls
Closure to pass to all callbacks in this struct.
#define GNUNET_YES
Definition: gnunet_common.h:77
int bulk_lock
Lock for bulk operations.
struct RIL_Peer_Agent * next
Next agent in solver&#39;s linked list.
#define LOG(kind,...)
GAS_bandwidth_changed_cb bandwidth_changed_cb
ATS addresses callback to be notified about bandwidth assignment changes.
After the problem was finished, notifications about changes to addresses are done.
struct ATS_Address * suggestion_address
The address which has to be issued.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ ril_step_scheduler_task()

static void ril_step_scheduler_task ( void *  cls)
static

Task for the scheduler, which performs one step and lets the solver know that no further step is scheduled.

Parameters
clsthe solver handle

Definition at line 1586 of file plugin_ats_ril.c.

References ril_step(), and GAS_RIL_Handle::step_next_task_id.

Referenced by ril_step_schedule_next().

1587 {
1588  struct GAS_RIL_Handle *solver = cls;
1589 
1590  solver->step_next_task_id = NULL;
1591  ril_step (solver);
1592 }
struct GNUNET_SCHEDULER_Task * step_next_task_id
Task identifier of the next time-step to be executed.
A handle for the reinforcement learning solver.
static void ril_step(struct GAS_RIL_Handle *solver)
Prototype of the ril_step() procedure.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ ril_get_used_resource_ratio()

static double ril_get_used_resource_ratio ( struct GAS_RIL_Handle solver)
static

Determines how much of the available bandwidth is assigned.

If more is assigned than available it returns 1. The function is used to determine the step size of the adaptive stepping.

Parameters
solverthe solver handle
Returns
the ratio

Definition at line 1603 of file plugin_ats_ril.c.

References RIL_Scope::bw_in_assigned, RIL_Scope::bw_in_available, RIL_Scope::bw_in_utilized, RIL_Scope::bw_out_available, RIL_Scope::bw_out_utilized, GAS_RIL_Handle::network_entries, and GAS_RIL_Handle::networks_count.

Referenced by ril_step_schedule_next().

1604 {
1605  int i;
1606  struct RIL_Scope net;
1607  unsigned long long sum_assigned = 0;
1608  unsigned long long sum_available = 0;
1609  double ratio;
1610 
1611  for (i = 0; i < solver->networks_count; i++)
1612  {
1613  net = solver->network_entries[i];
1614  if (net.bw_in_assigned > 0) // only consider scopes where an address is actually active
1615  {
1616  sum_assigned += net.bw_in_utilized;
1617  sum_assigned += net.bw_out_utilized;
1618  sum_available += net.bw_in_available;
1619  sum_available += net.bw_out_available;
1620  }
1621  }
1622  if (sum_available > 0)
1623  {
1624  ratio = ((double) sum_assigned) / ((double) sum_available);
1625  }
1626  else
1627  {
1628  ratio = 0;
1629  }
1630 
1631  return ratio > 1 ? 1 : ratio; // overutilization is possible, cap at 1
1632 }
struct RIL_Scope * network_entries
Array of networks with global assignment state.
uint32_t bw_in_utilized
Bandwidth inbound actually utilized in the network.
unsigned int networks_count
Networks count.
Here is the caller graph for this function:

◆ ril_get_network()

static struct RIL_Scope* ril_get_network ( struct GAS_RIL_Handle s,
uint32_t  type 
)
static

Lookup network struct by type.

Parameters
sthe solver handle
typethe network type
Returns
the network struct

Definition at line 1642 of file plugin_ats_ril.c.

References GAS_RIL_Handle::network_entries, GAS_RIL_Handle::networks_count, and RIL_Scope::type.

Referenced by GAS_ril_address_add(), ril_network_is_active(), and ril_network_is_not_full().

1643 {
1644  int i;
1645 
1646  for (i = 0; i < s->networks_count; i++)
1647  {
1648  if (s->network_entries[i].type == type)
1649  {
1650  return &s->network_entries[i];
1651  }
1652  }
1653  return NULL;
1654 }
struct RIL_Scope * network_entries
Array of networks with global assignment state.
enum GNUNET_NetworkType type
ATS network type.
unsigned int networks_count
Networks count.
enum GNUNET_TESTBED_UnderlayLinkModelType type
the type of this model
Here is the caller graph for this function:

◆ ril_network_is_not_full()

static int ril_network_is_not_full ( struct GAS_RIL_Handle solver,
enum GNUNET_NetworkType  network 
)
static

Determines whether more connections are allocated in a network scope, than they would theoretically fit.

This is used as a heuristic to determine, whether a new connection can be allocated or not.

Parameters
solverthe solver handle
networkthe network scope in question
Returns
GNUNET_YES if there are theoretically enough resources left

Definition at line 1666 of file plugin_ats_ril.c.

References address_count, RIL_Peer_Agent::address_inuse, GAS_RIL_Handle::agents_head, RIL_Scope::bw_in_available, RIL_Scope::bw_out_available, RIL_Peer_Agent::is_active, RIL_Peer_Agent::next, ril_get_network(), RIL_MIN_BW, ATS_Address::solver_information, and RIL_Scope::type.

Referenced by ril_try_unblock_agent().

1668 {
1669  struct RIL_Scope *net;
1670  struct RIL_Peer_Agent *agent;
1671  unsigned long long address_count = 0;
1672 
1673  for (agent = solver->agents_head; NULL != agent; agent = agent->next)
1674  {
1675  if (agent->address_inuse && agent->is_active)
1676  {
1677  net = agent->address_inuse->solver_information;
1678  if (net->type == network)
1679  {
1680  address_count++;
1681  }
1682  }
1683  }
1684 
1685  net = ril_get_network (solver, network);
1686  return (net->bw_in_available > RIL_MIN_BW * address_count) &&
1688 }
static struct RIL_Scope * ril_get_network(struct GAS_RIL_Handle *s, uint32_t type)
Lookup network struct by type.
struct RIL_Peer_Agent * agents_head
List of active peer-agents.
int is_active
Whether the agent is active or not.
enum GNUNET_NetworkType type
ATS network type.
static int address_count
Definition: gnunet-hello.c:50
void * solver_information
Solver-specific information for this address.
struct ATS_Address * address_inuse
Address in use.
uint32_t bw_out_available
Total available outbound bandwidth.
struct RIL_Peer_Agent * next
Next agent in solver&#39;s linked list.
uint32_t bw_in_available
Total available inbound bandwidth.
#define RIL_MIN_BW
Here is the call graph for this function:
Here is the caller graph for this function:

◆ ril_try_unblock_agent()

static void ril_try_unblock_agent ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent,
int  silent 
)
static

Unblocks an agent for which a connection request is there, that could not be satisfied.

Iterates over the addresses of the agent, if one of its addresses can now be allocated in its scope the agent is unblocked, otherwise it remains unchanged.

Parameters
solverthe solver handle
agentthe agent in question
silent

Definition at line 1701 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, RIL_Address_Wrapped::address_naked, RIL_Peer_Agent::addresses_head, RIL_Scope::bw_in_available, RIL_Scope::bw_in_utilized, RIL_Scope::bw_out_available, RIL_Scope::bw_out_utilized, envi_set_active_suggestion(), RIL_Address_Wrapped::next, RIL_MIN_BW, ril_network_is_not_full(), ATS_Address::solver_information, and RIL_Scope::type.

Referenced by GAS_ril_address_add(), GAS_ril_get_preferred_address(), and ril_step().

1703 {
1704  struct RIL_Address_Wrapped *addr_wrap;
1705  struct RIL_Scope *net;
1706  unsigned long long start_in;
1707  unsigned long long start_out;
1708 
1709  for (addr_wrap = agent->addresses_head; NULL != addr_wrap; addr_wrap =
1710  addr_wrap->next)
1711  {
1712  net = addr_wrap->address_naked->solver_information;
1713  if (ril_network_is_not_full (solver, net->type))
1714  {
1715  if (NULL == agent->address_inuse)
1716  {
1717  start_in = net->bw_in_available < net->bw_in_utilized ?
1718  (net->bw_in_available - net->bw_in_utilized) / 2 :
1719  RIL_MIN_BW;
1720  start_out = net->bw_out_available < net->bw_out_utilized ?
1721  (net->bw_out_available - net->bw_out_utilized) / 2 :
1722  RIL_MIN_BW;
1723  envi_set_active_suggestion (solver, agent, addr_wrap->address_naked,
1724  start_in, start_out, silent);
1725  }
1726  return;
1727  }
1728  }
1729  agent->address_inuse = NULL;
1730 }
struct RIL_Address_Wrapped * addresses_head
Head of addresses DLL.
static int ril_network_is_not_full(struct GAS_RIL_Handle *solver, enum GNUNET_NetworkType network)
Determines whether more connections are allocated in a network scope, than they would theoretically f...
static void envi_set_active_suggestion(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, struct ATS_Address *new_address, unsigned long long new_bw_in, unsigned long long new_bw_out, int silent)
Changes the active assignment suggestion of the handler and invokes the bw_changed callback to notify...
struct ATS_Address * address_naked
The address.
unsigned long long bw_out_utilized
Bandwidth outbound actually utilized in the network.
static int silent
–silent option
Definition: gnunet-qr.c:49
enum GNUNET_NetworkType type
ATS network type.
uint32_t bw_in_utilized
Bandwidth inbound actually utilized in the network.
void * solver_information
Solver-specific information for this address.
struct ATS_Address * address_inuse
Address in use.
uint32_t bw_out_available
Total available outbound bandwidth.
struct RIL_Address_Wrapped * next
Next in DLL.
Wrapper for addresses to store them in agent&#39;s linked list.
uint32_t bw_in_available
Total available inbound bandwidth.
#define RIL_MIN_BW
Here is the call graph for this function:
Here is the caller graph for this function:

◆ ril_calculate_discount()

static void ril_calculate_discount ( struct GAS_RIL_Handle solver)
static

Determines how much the reward needs to be discounted depending on the amount of time, which has passed since the last time-step.

Parameters
solverthe solver handle

Definition at line 1739 of file plugin_ats_ril.c.

References RIL_Learning_Parameters::beta, RIL_Learning_Parameters::gamma, GAS_RIL_Handle::global_discount_integrated, GAS_RIL_Handle::global_discount_variable, GNUNET_TIME_absolute_get(), GNUNET_TIME_absolute_get_difference(), GAS_RIL_Handle::parameters, GNUNET_TIME_Relative::rel_value_us, GAS_RIL_Handle::simulate, GAS_RIL_Handle::step_time_last, and RIL_Learning_Parameters::step_time_min.

Referenced by ril_step().

1740 {
1741  struct GNUNET_TIME_Absolute time_now;
1742  struct GNUNET_TIME_Relative time_delta;
1743  double tau;
1744 
1745  // MDP case only for debugging purposes
1746  if (solver->simulate)
1747  {
1748  solver->global_discount_variable = solver->parameters.gamma;
1749  solver->global_discount_integrated = 1;
1750  return;
1751  }
1752 
1753  // semi-MDP case
1754 
1755  // calculate tau, i.e. how many real valued time units have passed, one time unit is one minimum time step
1756  time_now = GNUNET_TIME_absolute_get ();
1758  time_now);
1759  solver->step_time_last = time_now;
1760  tau = (double) time_delta.rel_value_us
1761  / (double) solver->parameters.step_time_min.rel_value_us;
1762 
1763  // calculate reward discounts (once per step for all agents)
1764  solver->global_discount_variable = pow (M_E, ((-1.0)
1765  * ((double) solver->parameters.
1766  beta) * tau));
1767  solver->global_discount_integrated = (1.0 - solver->global_discount_variable)
1768  / (double) solver->parameters.beta;
1769 }
double global_discount_integrated
Integrated variable discount factor, dependent on time between steps.
uint64_t rel_value_us
The actual value.
double gamma
Learning discount factor in the TD-update for MDPs.
struct GNUNET_TIME_Absolute step_time_last
Timestamp for the last time-step.
unsigned long long simulate
Simulate steps, i.e.
double beta
Learning discount variable in the TD-update for semi-MDPs.
struct RIL_Learning_Parameters parameters
Learning parameters.
struct GNUNET_TIME_Absolute GNUNET_TIME_absolute_get(void)
Get the current time.
Definition: time.c:118
double global_discount_variable
Variable discount factor, dependent on time between steps.
Time for absolute times used by GNUnet, in microseconds.
struct GNUNET_TIME_Relative GNUNET_TIME_absolute_get_difference(struct GNUNET_TIME_Absolute start, struct GNUNET_TIME_Absolute end)
Compute the time difference between the given start and end times.
Definition: time.c:353
static float beta
Percentage of total peer number in the view to send random PULLs to.
struct GNUNET_TIME_Relative step_time_min
Minimal interval time between steps in milliseconds.
Time for relative time used by GNUnet, in microseconds.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ ril_network_count_active_agents()

static int ril_network_count_active_agents ( struct GAS_RIL_Handle solver,
struct RIL_Scope scope 
)
static

Count the number of active agents/connections in a network scope.

Parameters
solverthe solver handle
scopethe network scope in question
Returns
the number of allocated connections

Definition at line 1779 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, GAS_RIL_Handle::agents_head, RIL_Peer_Agent::is_active, RIL_Peer_Agent::next, and ATS_Address::solver_information.

Referenced by ril_networks_update_state().

1781 {
1782  int c = 0;
1783  struct RIL_Peer_Agent *cur_agent;
1784 
1785  for (cur_agent = solver->agents_head; NULL != cur_agent; cur_agent =
1786  cur_agent->next)
1787  {
1788  if (cur_agent->is_active && cur_agent->address_inuse &&
1789  (cur_agent->address_inuse->solver_information == scope))
1790  {
1791  c++;
1792  }
1793  }
1794  return c;
1795 }
struct RIL_Peer_Agent * agents_head
List of active peer-agents.
int is_active
Whether the agent is active or not.
void * solver_information
Solver-specific information for this address.
struct ATS_Address * address_inuse
Address in use.
struct RIL_Peer_Agent * next
Next agent in solver&#39;s linked list.
Here is the caller graph for this function:

◆ ril_network_get_assigned()

static unsigned long long ril_network_get_assigned ( struct GAS_RIL_Handle solver,
enum GNUNET_NetworkType  type,
int  direction_in 
)
static

Calculates how much bandwidth is assigned in sum in a network scope, either in the inbound or in the outbound direction.

Parameters
solverthe solver handle
typethe type of the network scope in question
direction_inGNUNET_YES if the inbound direction should be summed up, otherwise the outbound direction will be summed up
Returns
the sum of the assigned bandwidths

Definition at line 1808 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, GAS_RIL_Handle::agents_head, RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, RIL_Peer_Agent::is_active, RIL_Peer_Agent::next, ATS_Address::solver_information, and RIL_Scope::type.

Referenced by ril_networks_update_state().

1810 {
1811  struct RIL_Peer_Agent *cur;
1812  struct RIL_Scope *net;
1813  unsigned long long sum = 0;
1814 
1815  for (cur = solver->agents_head; NULL != cur; cur = cur->next)
1816  {
1817  if (cur->is_active && cur->address_inuse)
1818  {
1819  net = cur->address_inuse->solver_information;
1820  if (net->type == type)
1821  {
1822  if (direction_in)
1823  sum += cur->bw_in;
1824  else
1825  sum += cur->bw_out;
1826  }
1827  }
1828  }
1829 
1830  return sum;
1831 }
struct RIL_Peer_Agent * agents_head
List of active peer-agents.
uint32_t bw_in
Inbound bandwidth assigned by the agent.
int is_active
Whether the agent is active or not.
enum GNUNET_NetworkType type
ATS network type.
void * solver_information
Solver-specific information for this address.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
enum GNUNET_TESTBED_UnderlayLinkModelType type
the type of this model
struct RIL_Peer_Agent * next
Next agent in solver&#39;s linked list.
Here is the caller graph for this function:

◆ ril_network_get_utilized()

static unsigned long long ril_network_get_utilized ( struct GAS_RIL_Handle solver,
enum GNUNET_NetworkType  type,
int  direction_in 
)
static

Calculates how much bandwidth is actually utilized in sum in a network scope, either in the inbound or in the outbound direction.

Parameters
solverthe solver handle
typethe type of the network scope in question
direction_inGNUNET_YES if the inbound direction should be summed up, otherwise the outbound direction will be summed up
Returns
the sum of the utilized bandwidths (in bytes/second)

Definition at line 1844 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, GAS_RIL_Handle::agents_head, RIL_Peer_Agent::is_active, RIL_Peer_Agent::next, GAS_NormalizationInfo::norm, ATS_Address::norm_utilization_in, ATS_Address::norm_utilization_out, ATS_Address::solver_information, and RIL_Scope::type.

Referenced by ril_networks_update_state().

1846 {
1847  struct RIL_Peer_Agent *cur;
1848  struct RIL_Scope *net;
1849  unsigned long long sum = 0;
1850 
1851  for (cur = solver->agents_head; NULL != cur; cur = cur->next)
1852  {
1853  if (cur->is_active && cur->address_inuse)
1854  {
1855  net = cur->address_inuse->solver_information;
1856  if (net->type == type)
1857  {
1858  if (direction_in)
1859  sum += cur->address_inuse->norm_utilization_in.norm;
1860  else
1862  }
1863  }
1864  }
1865 
1866  return sum;
1867 }
struct RIL_Peer_Agent * agents_head
List of active peer-agents.
int is_active
Whether the agent is active or not.
enum GNUNET_NetworkType type
ATS network type.
struct GAS_NormalizationInfo norm_utilization_out
Normalized utilization outbound for this address.
void * solver_information
Solver-specific information for this address.
struct ATS_Address * address_inuse
Address in use.
struct GAS_NormalizationInfo norm_utilization_in
Normalized utilization inbound for this address.
enum GNUNET_TESTBED_UnderlayLinkModelType type
the type of this model
struct RIL_Peer_Agent * next
Next agent in solver&#39;s linked list.
double norm
Normalized values from queue to a range of values [1.0...2.0].
Here is the caller graph for this function:

◆ ril_networks_update_state()

static void ril_networks_update_state ( struct GAS_RIL_Handle solver)
static

Retrieves the state of the network scope, so that its attributes are up-to- date.

Parameters
solverthe solver handle

Definition at line 1876 of file plugin_ats_ril.c.

References RIL_Scope::active_agent_count, RIL_Scope::bw_in_assigned, RIL_Scope::bw_in_utilized, RIL_Scope::bw_out_assigned, RIL_Scope::bw_out_utilized, GNUNET_NO, GNUNET_YES, GAS_RIL_Handle::network_entries, GAS_RIL_Handle::networks_count, ril_network_count_active_agents(), ril_network_get_assigned(), ril_network_get_social_welfare(), ril_network_get_utilized(), RIL_Scope::social_welfare, and RIL_Scope::type.

Referenced by ril_step().

1877 {
1878  int c;
1879  struct RIL_Scope *net;
1880 
1881  for (c = 0; c < solver->networks_count; c++)
1882  {
1883  net = &solver->network_entries[c];
1884  net->bw_in_assigned = ril_network_get_assigned (solver, net->type,
1885  GNUNET_YES);
1886  net->bw_in_utilized = ril_network_get_utilized (solver, net->type,
1887  GNUNET_YES);
1888  net->bw_out_assigned = ril_network_get_assigned (solver, net->type,
1889  GNUNET_NO);
1890  net->bw_out_utilized = ril_network_get_utilized (solver, net->type,
1891  GNUNET_NO);
1893  net->social_welfare = ril_network_get_social_welfare (solver, net);
1894  }
1895 }
#define GNUNET_NO
Definition: gnunet_common.h:78
struct RIL_Scope * network_entries
Array of networks with global assignment state.
unsigned long long bw_out_utilized
Bandwidth outbound actually utilized in the network.
enum GNUNET_NetworkType type
ATS network type.
static unsigned long long ril_network_get_utilized(struct GAS_RIL_Handle *solver, enum GNUNET_NetworkType type, int direction_in)
Calculates how much bandwidth is actually utilized in sum in a network scope, either in the inbound o...
uint32_t bw_in_utilized
Bandwidth inbound actually utilized in the network.
double social_welfare
The social welfare achieved in the scope.
static int ril_network_count_active_agents(struct GAS_RIL_Handle *solver, struct RIL_Scope *scope)
Count the number of active agents/connections in a network scope.
unsigned int networks_count
Networks count.
static unsigned long long ril_network_get_assigned(struct GAS_RIL_Handle *solver, enum GNUNET_NetworkType type, int direction_in)
Calculates how much bandwidth is assigned in sum in a network scope, either in the inbound or in the ...
unsigned int active_agent_count
Number of active agents in scope.
uint32_t bw_in_assigned
Bandwidth inbound assigned in network after last step.
unsigned long long bw_out_assigned
Bandwidth outbound assigned in network after last step.
#define GNUNET_YES
Definition: gnunet_common.h:77
static double ril_network_get_social_welfare(struct GAS_RIL_Handle *solver, struct RIL_Scope *scope)
Calculates the social welfare within a network scope according to what social welfare measure is set ...
Here is the call graph for this function:
Here is the caller graph for this function:

◆ ril_step_schedule_next()

static void ril_step_schedule_next ( struct GAS_RIL_Handle solver)
static

Schedules the next global step in an adaptive way.

The more resources are left, the earlier the next step is scheduled. This serves the reactivity of the solver to changed inputs.

Parameters
solverthe solver handle

Definition at line 1905 of file plugin_ats_ril.c.

References GAS_RIL_Handle::done, GNUNET_assert, GNUNET_NO, GNUNET_SCHEDULER_add_delayed(), GNUNET_TIME_relative_saturating_multiply(), GNUNET_TIME_relative_subtract(), GNUNET_TIME_UNIT_MICROSECONDS, GNUNET_TIME_UNIT_ZERO, GAS_RIL_Handle::parameters, GNUNET_TIME_Relative::rel_value_us, ril_get_used_resource_ratio(), RIL_INTERVAL_EXPONENT, ril_step_scheduler_task(), GAS_RIL_Handle::simulate, GAS_RIL_Handle::step_next_task_id, RIL_Learning_Parameters::step_time_max, and RIL_Learning_Parameters::step_time_min.

Referenced by ril_step().

1906 {
1907  double used_ratio;
1908  double factor;
1909  double y;
1910  double offset;
1911  struct GNUNET_TIME_Relative time_next;
1912 
1913  used_ratio = ril_get_used_resource_ratio (solver);
1914 
1915  GNUNET_assert (
1918 
1919  factor = (double) GNUNET_TIME_relative_subtract (
1920  solver->parameters.step_time_max,
1921  solver->parameters.
1922  step_time_min).rel_value_us;
1923  offset = (double) solver->parameters.step_time_min.rel_value_us;
1924  y = factor * pow (used_ratio, RIL_INTERVAL_EXPONENT) + offset;
1925 
1926  GNUNET_assert (y <= (double) solver->parameters.step_time_max.rel_value_us);
1927  GNUNET_assert (y >= (double) solver->parameters.step_time_min.rel_value_us);
1928 
1930  GNUNET_TIME_UNIT_MICROSECONDS, (unsigned long long) y);
1931 
1932 // LOG (GNUNET_ERROR_TYPE_INFO, "ratio: %f, factor: %f, offset: %f, y: %f\n",
1933 // used_ratio,
1934 // factor,
1935 // offset,
1936 // y);
1937 
1938  if (solver->simulate)
1939  {
1940  time_next = GNUNET_TIME_UNIT_ZERO;
1941  }
1942 
1943  if ((NULL == solver->step_next_task_id) && (GNUNET_NO == solver->done))
1944  {
1945  solver->step_next_task_id = GNUNET_SCHEDULER_add_delayed (time_next,
1946  &
1948  solver);
1949  }
1950 }
struct GNUNET_SCHEDULER_Task * step_next_task_id
Task identifier of the next time-step to be executed.
uint64_t rel_value_us
The actual value.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
struct GNUNET_TIME_Relative step_time_max
Maximum interval time between steps in milliseconds.
#define GNUNET_NO
Definition: gnunet_common.h:78
#define RIL_INTERVAL_EXPONENT
static void ril_step_scheduler_task(void *cls)
Task for the scheduler, which performs one step and lets the solver know that no further step is sche...
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_delayed(struct GNUNET_TIME_Relative delay, GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run with a specified delay.
Definition: scheduler.c:1253
unsigned long long simulate
Simulate steps, i.e.
struct GNUNET_TIME_Relative GNUNET_TIME_relative_subtract(struct GNUNET_TIME_Relative a1, struct GNUNET_TIME_Relative a2)
Subtract relative timestamp from the other.
Definition: time.c:602
struct RIL_Learning_Parameters parameters
Learning parameters.
#define GNUNET_TIME_UNIT_ZERO
Relative time zero.
int done
Shutdown.
static double ril_get_used_resource_ratio(struct GAS_RIL_Handle *solver)
Determines how much of the available bandwidth is assigned.
struct GNUNET_TIME_Relative step_time_min
Minimal interval time between steps in milliseconds.
#define GNUNET_TIME_UNIT_MICROSECONDS
One microsecond, our basic time unit.
struct GNUNET_TIME_Relative GNUNET_TIME_relative_saturating_multiply(struct GNUNET_TIME_Relative rel, unsigned long long factor)
Saturating multiply relative time by a given factor.
Definition: time.c:499
Time for relative time used by GNUnet, in microseconds.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_w_init()

static void agent_w_init ( struct RIL_Peer_Agent agent)
static

Initializes the matrix W of parameter vectors theta with small random numbers.

Parameters
agentThe respective agent

Definition at line 2022 of file plugin_ats_ril.c.

References RIL_Learning_Parameters::alpha, RIL_Peer_Agent::envi, GNUNET_CRYPTO_QUALITY_WEAK, GNUNET_CRYPTO_random_u32(), RIL_Peer_Agent::m, RIL_Peer_Agent::n, GAS_RIL_Handle::parameters, and RIL_Peer_Agent::W.

Referenced by agent_init().

2023 {
2024  int i;
2025  int k;
2026 
2027  for (i = 0; i < agent->n; i++)
2028  {
2029  for (k = 0; k < agent->m; k++)
2030  {
2031  agent->W[i][k] = agent->envi->parameters.alpha * (1.0 - 2.0
2032  * ((double)
2035  UINT32_MAX)
2036  / (double) UINT32_MAX));
2037  }
2038  }
2039 }
unsigned int m
Number of rows of W / Number of state-vector features.
uint32_t GNUNET_CRYPTO_random_u32(enum GNUNET_CRYPTO_Quality mode, uint32_t i)
Produce a random value.
double ** W
Experience matrix W.
struct RIL_Learning_Parameters parameters
Learning parameters.
unsigned int n
Number of columns of W / Number of actions.
struct GAS_RIL_Handle * envi
Environment handle.
No good quality of the operation is needed (i.e., random numbers can be pseudo-random).
double alpha
Gradient-descent step-size.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_init()

static struct RIL_Peer_Agent* agent_init ( void *  s,
const struct GNUNET_PeerIdentity peer 
)
static

Initialize an agent without addresses and its knowledge base.

Parameters
sril solver
peerthe one in question
Returns
handle to the new agent

Definition at line 2049 of file plugin_ats_ril.c.

References RIL_Peer_Agent::a_old, RIL_Peer_Agent::address_inuse, agent_w_init(), RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, RIL_Peer_Agent::E, RIL_Peer_Agent::eligibility_reset, RIL_Peer_Agent::envi, GNUNET_malloc, GNUNET_new, GNUNET_NO, RIL_Peer_Agent::is_active, RIL_Peer_Agent::m, RIL_Peer_Agent::n, RIL_Peer_Agent::nop_bonus, RIL_Peer_Agent::objective_old, peer, RIL_Peer_Agent::peer, RIL_ACTION_INVALID, RIL_ACTION_TYPE_NUM, RIL_MIN_BW, RIL_Peer_Agent::s_old, RIL_Peer_Agent::step_count, RIL_Peer_Agent::suggestion_issue, and RIL_Peer_Agent::W.

Referenced by ril_get_agent().

2050 {
2051  int i;
2052  struct GAS_RIL_Handle *solver = s;
2053  struct RIL_Peer_Agent *agent = GNUNET_new (struct RIL_Peer_Agent);
2054 
2055  agent->envi = solver;
2056  agent->peer = *peer;
2057  agent->step_count = 0;
2058  agent->is_active = GNUNET_NO;
2059  agent->bw_in = RIL_MIN_BW;
2060  agent->bw_out = RIL_MIN_BW;
2061  agent->suggestion_issue = GNUNET_NO;
2062  agent->n = RIL_ACTION_TYPE_NUM;
2063  agent->m = 0;
2064  agent->W = (double **) GNUNET_malloc (sizeof(double *) * agent->n);
2065  agent->E = (double **) GNUNET_malloc (sizeof(double *) * agent->n);
2066  for (i = 0; i < agent->n; i++)
2067  {
2068  agent->W[i] = (double *) GNUNET_malloc (sizeof(double) * agent->m);
2069  agent->E[i] = (double *) GNUNET_malloc (sizeof(double) * agent->m);
2070  }
2071  agent_w_init (agent);
2072  agent->eligibility_reset = GNUNET_NO;
2073  agent->a_old = RIL_ACTION_INVALID;
2074  agent->s_old = GNUNET_malloc (sizeof(double) * agent->m);
2075  agent->address_inuse = NULL;
2076  agent->objective_old = 0;
2077  agent->nop_bonus = 0;
2078 
2079  return agent;
2080 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
int is_active
Whether the agent is active or not.
unsigned int m
Number of rows of W / Number of state-vector features.
A handle for the reinforcement learning solver.
int eligibility_reset
Whether to reset the eligibility traces to 0 after a Q-exploration step.
int a_old
Last chosen action.
#define GNUNET_NO
Definition: gnunet_common.h:78
int suggestion_issue
Flag whether a suggestion has to be issued.
#define GNUNET_new(type)
Allocate a struct or union of the given type.
struct GNUNET_PeerIdentity peer
Peer ID.
static void agent_w_init(struct RIL_Peer_Agent *agent)
Initializes the matrix W of parameter vectors theta with small random numbers.
double ** W
Experience matrix W.
struct GNUNET_TESTBED_Peer * peer
The peer associated with this model.
double ** E
Eligibility traces.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
unsigned int n
Number of columns of W / Number of actions.
#define RIL_ACTION_INVALID
double * s_old
Last perceived state feature vector.
struct GAS_RIL_Handle * envi
Environment handle.
#define GNUNET_malloc(size)
Wrapper around malloc.
double nop_bonus
NOP bonus.
double objective_old
The agent&#39;s last objective value.
unsigned long long step_count
Number of performed time-steps.
#define RIL_MIN_BW
Here is the call graph for this function:
Here is the caller graph for this function:

◆ agent_die()

static void agent_die ( struct GAS_RIL_Handle solver,
struct RIL_Peer_Agent agent 
)
static

Deallocate agent.

Parameters
solverthe solver handle
agentthe agent to retire

Definition at line 2089 of file plugin_ats_ril.c.

References RIL_Peer_Agent::E, GNUNET_free, GNUNET_free_non_null, RIL_Peer_Agent::n, RIL_Peer_Agent::s_old, and RIL_Peer_Agent::W.

Referenced by libgnunet_plugin_ats_ril_done().

2090 {
2091  int i;
2092 
2093  for (i = 0; i < agent->n; i++)
2094  {
2095  GNUNET_free_non_null (agent->W[i]);
2096  GNUNET_free_non_null (agent->E[i]);
2097  }
2098  GNUNET_free_non_null (agent->W);
2099  GNUNET_free_non_null (agent->E);
2100  GNUNET_free_non_null (agent->s_old);
2101  GNUNET_free (agent);
2102 }
#define GNUNET_free_non_null(ptr)
Free the memory pointed to by ptr if ptr is not NULL.
double ** W
Experience matrix W.
double ** E
Eligibility traces.
unsigned int n
Number of columns of W / Number of actions.
double * s_old
Last perceived state feature vector.
#define GNUNET_free(ptr)
Wrapper around free.
Here is the caller graph for this function:

◆ ril_get_agent()

static struct RIL_Peer_Agent* ril_get_agent ( struct GAS_RIL_Handle solver,
const struct GNUNET_PeerIdentity peer,
int  create 
)
static

Returns the agent for a peer.

Parameters
solverthe solver handle
peerthe identity of the peer
createwhether or not to create an agent, if none is allocated yet
Returns
the agent

Definition at line 2113 of file plugin_ats_ril.c.

References agent_init(), GAS_RIL_Handle::agents_head, GAS_RIL_Handle::agents_tail, GNUNET_CONTAINER_DLL_insert_tail, GNUNET_memcmp, RIL_Peer_Agent::next, and RIL_Peer_Agent::peer.

Referenced by GAS_ril_address_add(), GAS_ril_address_delete(), GAS_ril_get_preferred_address(), and GAS_ril_stop_get_preferred_address().

2115 {
2116  struct RIL_Peer_Agent *cur;
2117 
2118  for (cur = solver->agents_head; NULL != cur; cur = cur->next)
2119  {
2120  if (0 == GNUNET_memcmp (peer, &cur->peer))
2121  {
2122  return cur;
2123  }
2124  }
2125 
2126  if (create)
2127  {
2128  cur = agent_init (solver, peer);
2130  cur);
2131  return cur;
2132  }
2133  return NULL;
2134 }
struct RIL_Peer_Agent * agents_head
List of active peer-agents.
struct RIL_Peer_Agent * agents_tail
struct GNUNET_PeerIdentity peer
Peer ID.
static struct RIL_Peer_Agent * agent_init(void *s, const struct GNUNET_PeerIdentity *peer)
Initialize an agent without addresses and its knowledge base.
#define GNUNET_CONTAINER_DLL_insert_tail(head, tail, element)
Insert an element at the tail of a DLL.
#define GNUNET_memcmp(a, b)
Compare memory in a and b, where both must be of the same pointer type.
struct RIL_Peer_Agent * next
Next agent in solver&#39;s linked list.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ ril_network_is_active()

static int ril_network_is_active ( struct GAS_RIL_Handle solver,
enum GNUNET_NetworkType  network 
)
static

Determine whether at least the minimum bandwidth is set for the network.

Otherwise the network is considered inactive and not used. Addresses in an inactive network are ignored.

Parameters
solversolver handle
networkthe network type
Returns
whether or not the network is considered active

Definition at line 2145 of file plugin_ats_ril.c.

References RIL_Scope::bw_out_available, ril_get_network(), and RIL_MIN_BW.

Referenced by GAS_ril_address_add(), and GAS_ril_address_delete().

2147 {
2148  struct RIL_Scope *net;
2149 
2150  net = ril_get_network (solver, network);
2151  return net->bw_out_available >= RIL_MIN_BW;
2152 }
static struct RIL_Scope * ril_get_network(struct GAS_RIL_Handle *s, uint32_t type)
Lookup network struct by type.
uint32_t bw_out_available
Total available outbound bandwidth.
#define RIL_MIN_BW
Here is the call graph for this function:
Here is the caller graph for this function:

◆ ril_cut_from_vector()

static void ril_cut_from_vector ( void **  old,
size_t  element_size,
unsigned int  hole_start,
unsigned int  hole_length,
unsigned int  old_length 
)
static

Cuts a slice out of a vector of elements.

This is used to decrease the size of the matrix storing the reward function approximation. It copies the memory, which is not cut, to the new vector, frees the memory of the old vector, and redirects the pointer to the new one.

Parameters
oldpointer to the pointer to the first element of the vector
element_sizebyte size of the vector elements
hole_startthe first element to cut out
hole_lengththe number of elements to cut out
old_lengththe length of the old vector

Definition at line 2166 of file plugin_ats_ril.c.

References GNUNET_assert, GNUNET_free, GNUNET_malloc, GNUNET_memcpy, and size.

Referenced by GAS_ril_address_delete().

2171 {
2172  char *tmpptr;
2173  char *oldptr = (char *) *old;
2174  size_t size;
2175  unsigned int bytes_before;
2176  unsigned int bytes_hole;
2177  unsigned int bytes_after;
2178 
2179  GNUNET_assert (old_length >= hole_length);
2180  GNUNET_assert (old_length >= (hole_start + hole_length));
2181 
2182  size = element_size * (old_length - hole_length);
2183 
2184  bytes_before = element_size * hole_start;
2185  bytes_hole = element_size * hole_length;
2186  bytes_after = element_size * (old_length - hole_start - hole_length);
2187 
2188  if (0 == size)
2189  {
2190  tmpptr = NULL;
2191  }
2192  else
2193  {
2194  tmpptr = GNUNET_malloc (size);
2195  GNUNET_memcpy (tmpptr, oldptr, bytes_before);
2196  GNUNET_memcpy (tmpptr + bytes_before, oldptr + (bytes_before + bytes_hole),
2197  bytes_after);
2198  }
2199  if (NULL != *old)
2200  {
2201  GNUNET_free (*old);
2202  }
2203  *old = (void *) tmpptr;
2204 }
static unsigned int element_size
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
#define GNUNET_memcpy(dst, src, n)
Call memcpy() but check for n being 0 first.
static unsigned int size
Size of the "table".
Definition: peer.c:67
#define GNUNET_malloc(size)
Wrapper around malloc.
#define GNUNET_free(ptr)
Wrapper around free.
Here is the caller graph for this function:

◆ GAS_ril_address_change_preference()

static void GAS_ril_address_change_preference ( void *  solver,
const struct GNUNET_PeerIdentity peer,
enum GNUNET_ATS_PreferenceKind  kind,
double  pref_rel 
)
static

Change relative preference for quality in solver.

Parameters
solverthe solver handle
peerthe peer to change the preference for
kindthe kind to change the preference
pref_relthe normalized preference value for this kind over all clients

Definition at line 2220 of file plugin_ats_ril.c.

References RIL_Learning_Parameters::epsilon, RIL_Learning_Parameters::epsilon_init, GNUNET_ATS_print_preference_type(), GNUNET_ERROR_TYPE_DEBUG, GNUNET_i2s(), LOG, GAS_RIL_Handle::parameters, ril_step(), RIL_Learning_Parameters::temperature, and RIL_Learning_Parameters::temperature_init.

Referenced by libgnunet_plugin_ats_ril_init().

2224 {
2226  "API_address_change_preference() Preference '%s' for peer '%s' changed to %.2f \n",
2227  GNUNET_ATS_print_preference_type (kind), GNUNET_i2s (peer), pref_rel);
2228 
2229  struct GAS_RIL_Handle *s = solver;
2230 
2233  ril_step (s);
2234 }
A handle for the reinforcement learning solver.
double epsilon
Ratio, with what probability an agent should explore in the e-greed policy.
double temperature
Softmax action-selection temperature.
const char * GNUNET_ATS_print_preference_type(enum GNUNET_ATS_PreferenceKind type)
Convert a enum GNUNET_ATS_PreferenceType to a string.
struct RIL_Learning_Parameters parameters
Learning parameters.
double epsilon_init
Initial exploration ratio value.
#define LOG(kind,...)
static void ril_step(struct GAS_RIL_Handle *solver)
Prototype of the ril_step() procedure.
const char * GNUNET_i2s(const struct GNUNET_PeerIdentity *pid)
Convert a peer identity to a string (for printing debug messages).
double temperature_init
Initial softmax action-selection temperature.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ GAS_ril_address_add()

static void GAS_ril_address_add ( void *  solver,
struct ATS_Address address,
uint32_t  network 
)
static

Add a new address for a peer to the solver.

The address is already contained in the addresses hashmap!

Parameters
solverthe solver Handle
addressthe address to add
networknetwork type of this address

Definition at line 2247 of file plugin_ats_ril.c.

References ATS_Address::active, ATS_Address::addr, address, RIL_Address_Wrapped::address_naked, RIL_Peer_Agent::addresses_head, RIL_Peer_Agent::addresses_tail, RIL_Peer_Agent::E, RIL_Learning_Parameters::epsilon, RIL_Learning_Parameters::epsilon_init, GNUNET_array_grow, GNUNET_CONTAINER_DLL_insert_tail, GNUNET_ERROR_TYPE_DEBUG, GNUNET_i2s(), GNUNET_new, GNUNET_NO, GNUNET_YES, LOG, RIL_Peer_Agent::m, RIL_Peer_Agent::n, GAS_RIL_Handle::parameters, ATS_Address::peer, ATS_Address::plugin, RIL_Learning_Parameters::rbf_divisor, ril_get_agent(), ril_get_network(), ril_network_is_active(), ril_step(), ril_try_unblock_agent(), RIL_Peer_Agent::s_old, ATS_Address::solver_information, RIL_Learning_Parameters::temperature, RIL_Learning_Parameters::temperature_init, RIL_Peer_Agent::W, and zero.

Referenced by libgnunet_plugin_ats_ril_init().

2250 {
2251  struct GAS_RIL_Handle *s = solver;
2252  struct RIL_Peer_Agent *agent;
2253  struct RIL_Address_Wrapped *address_wrapped;
2254  struct RIL_Scope *net;
2255  unsigned int m_new;
2256  unsigned int m_old;
2257  unsigned int n_new;
2258  unsigned int n_old;
2259  int i;
2260  unsigned int zero;
2261 
2263  "API_address_add()\n");
2264 
2265  net = ril_get_network (s, network);
2266  address->solver_information = net;
2267 
2268  if (! ril_network_is_active (s, network))
2269  {
2271  "API_address_add() Did not add %s address %s for peer '%s', network does not have enough bandwidth\n",
2272  address->plugin, address->addr, GNUNET_i2s (&address->peer));
2273  return;
2274  }
2275 
2278 
2279  agent = ril_get_agent (s, &address->peer, GNUNET_YES);
2280 
2281  // add address
2282  address_wrapped = GNUNET_new (struct RIL_Address_Wrapped);
2283  address_wrapped->address_naked = address;
2285  agent->addresses_tail, address_wrapped);
2286 
2287  // increase size of W
2288  m_new = agent->m + ((s->parameters.rbf_divisor + 1)
2289  * (s->parameters.rbf_divisor + 1));
2290  m_old = agent->m;
2291  n_new = agent->n + 1;
2292  n_old = agent->n;
2293 
2294  GNUNET_array_grow (agent->W, agent->n, n_new);
2295  agent->n = n_old;
2296  GNUNET_array_grow (agent->E, agent->n, n_new);
2297  for (i = 0; i < n_new; i++)
2298  {
2299  if (i < n_old)
2300  {
2301  agent->m = m_old;
2302  GNUNET_array_grow (agent->W[i], agent->m, m_new);
2303  agent->m = m_old;
2304  GNUNET_array_grow (agent->E[i], agent->m, m_new);
2305  }
2306  else
2307  {
2308  zero = 0;
2309  GNUNET_array_grow (agent->W[i], zero, m_new);
2310  zero = 0;
2311  GNUNET_array_grow (agent->E[i], zero, m_new);
2312  }
2313  }
2314 
2315  // increase size of old state vector
2316  agent->m = m_old;
2317  GNUNET_array_grow (agent->s_old, agent->m, m_new);
2318 
2319  ril_try_unblock_agent (s, agent, GNUNET_NO);
2320 
2321  ril_step (s);
2322 
2324  "API_address_add() Added %s %s address %s for peer '%s'\n",
2325  address->active ? "active" : "inactive", address->plugin, address->addr,
2326  GNUNET_i2s (&address->peer));
2327 }
static const struct GNUNET_CRYPTO_EcdsaPrivateKey zero
Public key of all zeros.
static struct RIL_Scope * ril_get_network(struct GAS_RIL_Handle *s, uint32_t type)
Lookup network struct by type.
int active
Is this the active address for this peer?
unsigned int m
Number of rows of W / Number of state-vector features.
struct GNUNET_PeerIdentity peer
Peer ID this address is for.
A handle for the reinforcement learning solver.
const void * addr
Address (in plugin-specific binary format).
struct RIL_Address_Wrapped * addresses_head
Head of addresses DLL.
#define GNUNET_NO
Definition: gnunet_common.h:78
char * plugin
Plugin name.
#define GNUNET_new(type)
Allocate a struct or union of the given type.
static void ril_try_unblock_agent(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int silent)
Unblocks an agent for which a connection request is there, that could not be satisfied.
double epsilon
Ratio, with what probability an agent should explore in the e-greed policy.
static int ril_network_is_active(struct GAS_RIL_Handle *solver, enum GNUNET_NetworkType network)
Determine whether at least the minimum bandwidth is set for the network.
struct ATS_Address * address_naked
The address.
double temperature
Softmax action-selection temperature.
#define GNUNET_array_grow(arr, size, tsize)
Grow a well-typed (!) array.
double ** W
Experience matrix W.
struct RIL_Learning_Parameters parameters
Learning parameters.
static struct RIL_Peer_Agent * ril_get_agent(struct GAS_RIL_Handle *solver, const struct GNUNET_PeerIdentity *peer, int create)
Returns the agent for a peer.
void * solver_information
Solver-specific information for this address.
double ** E
Eligibility traces.
#define GNUNET_CONTAINER_DLL_insert_tail(head, tail, element)
Insert an element at the tail of a DLL.
double epsilon_init
Initial exploration ratio value.
unsigned int n
Number of columns of W / Number of actions.
double * s_old
Last perceived state feature vector.
#define GNUNET_YES
Definition: gnunet_common.h:77
struct RIL_Address_Wrapped * addresses_tail
Tail of addresses DLL.
unsigned long long rbf_divisor
State space divisor.
#define LOG(kind,...)
static void ril_step(struct GAS_RIL_Handle *solver)
Prototype of the ril_step() procedure.
static char * address
GNS address for this phone.
const char * GNUNET_i2s(const struct GNUNET_PeerIdentity *pid)
Convert a peer identity to a string (for printing debug messages).
Wrapper for addresses to store them in agent&#39;s linked list.
double temperature_init
Initial softmax action-selection temperature.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ GAS_ril_address_delete()

static void GAS_ril_address_delete ( void *  solver,
struct ATS_Address address 
)
static

Delete an address in the solver.

The address is not contained in the address hashmap anymore!

Parameters
solverthe solver handle
addressthe address to remove

Definition at line 2338 of file plugin_ats_ril.c.

References RIL_Peer_Agent::a_old, ATS_Address::active, ATS_Address::addr, RIL_Peer_Agent::address_inuse, RIL_Address_Wrapped::address_naked, RIL_Peer_Agent::addresses_head, RIL_Peer_Agent::addresses_tail, agent_address_get_index(), agent_address_get_wrapped(), RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, RIL_Peer_Agent::E, envi_set_active_suggestion(), RIL_Learning_Parameters::epsilon, RIL_Learning_Parameters::epsilon_init, GNUNET_assert, GNUNET_CONTAINER_DLL_remove, GNUNET_ERROR_TYPE_DEBUG, GNUNET_free, GNUNET_free_non_null, GNUNET_i2s(), GNUNET_log, GNUNET_NO, GNUNET_YES, LOG, RIL_Peer_Agent::m, RIL_Peer_Agent::n, GAS_RIL_Handle::parameters, ATS_Address::peer, ATS_Address::plugin, RIL_Learning_Parameters::rbf_divisor, RIL_ACTION_INVALID, RIL_ACTION_TYPE_NUM, ril_cut_from_vector(), ril_get_agent(), ril_network_is_active(), ril_step(), RIL_Peer_Agent::s_old, ATS_Address::solver_information, RIL_Peer_Agent::suggestion_address, RIL_Peer_Agent::suggestion_issue, RIL_Learning_Parameters::temperature, RIL_Learning_Parameters::temperature_init, RIL_Scope::type, and RIL_Peer_Agent::W.

Referenced by libgnunet_plugin_ats_ril_init().

2340 {
2341  struct GAS_RIL_Handle *s = solver;
2342  struct RIL_Peer_Agent *agent;
2343  struct RIL_Address_Wrapped *address_wrapped;
2344  int address_index;
2345  unsigned int m_new;
2346  unsigned int n_new;
2347  int i;
2348  struct RIL_Scope *net;
2349 
2351  "API_address_delete() Delete %s %s address %s for peer '%s'\n",
2352  address->active ? "active" : "inactive",
2353  address->plugin,
2354  address->addr,
2355  GNUNET_i2s (&address->peer));
2356 
2357  agent = ril_get_agent (s, &address->peer, GNUNET_NO);
2358  if (NULL == agent)
2359  {
2360  net = address->solver_information;
2363  "No agent allocated for peer yet, since address was in inactive network\n");
2364  return;
2365  }
2366 
2369 
2370  address_index = agent_address_get_index (agent, address);
2371  address_wrapped = agent_address_get_wrapped (agent, address);
2372 
2373  if (NULL == address_wrapped)
2374  {
2375  net = address->solver_information;
2377  "Address not considered by agent, address was in inactive network\n");
2378  return;
2379  }
2381  agent->addresses_tail,
2382  address_wrapped);
2383  GNUNET_free (address_wrapped);
2384 
2385  // decrease W
2386  m_new = agent->m - ((s->parameters.rbf_divisor + 1)
2387  * (s->parameters.rbf_divisor + 1));
2388  n_new = agent->n - 1;
2389 
2390  for (i = 0; i < agent->n; i++)
2391  {
2392  ril_cut_from_vector ((void **) &agent->W[i], sizeof(double),
2393  address_index * ((s->parameters.rbf_divisor + 1)
2394  * (s->parameters.rbf_divisor + 1)),
2395  ((s->parameters.rbf_divisor + 1)
2396  * (s->parameters.rbf_divisor + 1)), agent->m);
2397  ril_cut_from_vector ((void **) &agent->E[i], sizeof(double),
2398  address_index * ((s->parameters.rbf_divisor + 1)
2399  * (s->parameters.rbf_divisor + 1)),
2400  ((s->parameters.rbf_divisor + 1)
2401  * (s->parameters.rbf_divisor + 1)), agent->m);
2402  }
2403  GNUNET_free_non_null (agent->W[RIL_ACTION_TYPE_NUM + address_index]);
2404  GNUNET_free_non_null (agent->E[RIL_ACTION_TYPE_NUM + address_index]);
2405  ril_cut_from_vector ((void **) &agent->W, sizeof(double *),
2406  RIL_ACTION_TYPE_NUM + address_index,
2407  1, agent->n);
2408  ril_cut_from_vector ((void **) &agent->E, sizeof(double *),
2409  RIL_ACTION_TYPE_NUM + address_index,
2410  1, agent->n);
2411  // correct last action
2412  if (agent->a_old > (RIL_ACTION_TYPE_NUM + address_index))
2413  {
2414  agent->a_old -= 1;
2415  }
2416  else if (agent->a_old == (RIL_ACTION_TYPE_NUM + address_index))
2417  {
2418  agent->a_old = RIL_ACTION_INVALID;
2419  }
2420  // decrease old state vector
2421  ril_cut_from_vector ((void **) &agent->s_old, sizeof(double),
2422  address_index * ((s->parameters.rbf_divisor + 1)
2423  * (s->parameters.rbf_divisor + 1)),
2424  ((s->parameters.rbf_divisor + 1)
2425  * (s->parameters.rbf_divisor + 1)), agent->m);
2426  agent->m = m_new;
2427  agent->n = n_new;
2428 
2429  if (agent->address_inuse == address)
2430  {
2431  if (NULL != agent->addresses_head) // if peer has an address left, use it
2432  {
2434  "Active address died, suggesting alternative!\n");
2436  agent,
2437  agent->addresses_head->address_naked,
2438  agent->bw_in,
2439  agent->bw_out,
2440  GNUNET_YES);
2441  }
2442  else
2443  {
2445  "Active address died, suggesting disconnect!\n");
2446  envi_set_active_suggestion (s, agent, NULL, 0, 0, GNUNET_NO);
2447  }
2448  }
2449  ril_step (solver);
2450  if (agent->suggestion_address == address)
2451  {
2452  agent->suggestion_issue = GNUNET_NO;
2453  agent->suggestion_address = NULL;
2454  }
2455  GNUNET_assert (agent->address_inuse != address);
2456 }
#define GNUNET_CONTAINER_DLL_remove(head, tail, element)
Remove an element from a DLL.
uint32_t bw_in
Inbound bandwidth assigned by the agent.
int active
Is this the active address for this peer?
unsigned int m
Number of rows of W / Number of state-vector features.
struct GNUNET_PeerIdentity peer
Peer ID this address is for.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
A handle for the reinforcement learning solver.
const void * addr
Address (in plugin-specific binary format).
struct RIL_Address_Wrapped * addresses_head
Head of addresses DLL.
int a_old
Last chosen action.
static void ril_cut_from_vector(void **old, size_t element_size, unsigned int hole_start, unsigned int hole_length, unsigned int old_length)
Cuts a slice out of a vector of elements.
#define GNUNET_NO
Definition: gnunet_common.h:78
#define GNUNET_free_non_null(ptr)
Free the memory pointed to by ptr if ptr is not NULL.
char * plugin
Plugin name.
int suggestion_issue
Flag whether a suggestion has to be issued.
double epsilon
Ratio, with what probability an agent should explore in the e-greed policy.
static int ril_network_is_active(struct GAS_RIL_Handle *solver, enum GNUNET_NetworkType network)
Determine whether at least the minimum bandwidth is set for the network.
static void envi_set_active_suggestion(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, struct ATS_Address *new_address, unsigned long long new_bw_in, unsigned long long new_bw_out, int silent)
Changes the active assignment suggestion of the handler and invokes the bw_changed callback to notify...
struct ATS_Address * address_naked
The address.
double temperature
Softmax action-selection temperature.
enum GNUNET_NetworkType type
ATS network type.
static int agent_address_get_index(struct RIL_Peer_Agent *agent, struct ATS_Address *address)
Get the index of the address in the agent&#39;s list.
double ** W
Experience matrix W.
struct RIL_Learning_Parameters parameters
Learning parameters.
static struct RIL_Peer_Agent * ril_get_agent(struct GAS_RIL_Handle *solver, const struct GNUNET_PeerIdentity *peer, int create)
Returns the agent for a peer.
void * solver_information
Solver-specific information for this address.
double ** E
Eligibility traces.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
static struct RIL_Address_Wrapped * agent_address_get_wrapped(struct RIL_Peer_Agent *agent, struct ATS_Address *address)
Gets the wrapped address from the agent&#39;s list.
struct ATS_Address * address_inuse
Address in use.
double epsilon_init
Initial exploration ratio value.
unsigned int n
Number of columns of W / Number of actions.
#define RIL_ACTION_INVALID
#define GNUNET_log(kind,...)
double * s_old
Last perceived state feature vector.
#define GNUNET_YES
Definition: gnunet_common.h:77
struct RIL_Address_Wrapped * addresses_tail
Tail of addresses DLL.
unsigned long long rbf_divisor
State space divisor.
#define LOG(kind,...)
static void ril_step(struct GAS_RIL_Handle *solver)
Prototype of the ril_step() procedure.
struct ATS_Address * suggestion_address
The address which has to be issued.
const char * GNUNET_i2s(const struct GNUNET_PeerIdentity *pid)
Convert a peer identity to a string (for printing debug messages).
Wrapper for addresses to store them in agent&#39;s linked list.
double temperature_init
Initial softmax action-selection temperature.
#define GNUNET_free(ptr)
Wrapper around free.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ GAS_ril_address_property_changed()

static void GAS_ril_address_property_changed ( void *  solver,
struct ATS_Address address 
)
static

Update the properties of an address in the solver.

Parameters
solversolver handle
addressthe address

Definition at line 2466 of file plugin_ats_ril.c.

References RIL_Learning_Parameters::epsilon, RIL_Learning_Parameters::epsilon_init, GNUNET_ERROR_TYPE_DEBUG, GNUNET_i2s(), LOG, GAS_RIL_Handle::parameters, ATS_Address::peer, ril_step(), RIL_Learning_Parameters::temperature, and RIL_Learning_Parameters::temperature_init.

Referenced by libgnunet_plugin_ats_ril_init().

2468 {
2469  struct GAS_RIL_Handle *s = solver;
2470 
2472  "Properties for peer '%s' address changed\n",
2473  GNUNET_i2s (&address->peer));
2476  ril_step (s);
2477 }
struct GNUNET_PeerIdentity peer
Peer ID this address is for.
A handle for the reinforcement learning solver.
double epsilon
Ratio, with what probability an agent should explore in the e-greed policy.
double temperature
Softmax action-selection temperature.
struct RIL_Learning_Parameters parameters
Learning parameters.
double epsilon_init
Initial exploration ratio value.
#define LOG(kind,...)
static void ril_step(struct GAS_RIL_Handle *solver)
Prototype of the ril_step() procedure.
const char * GNUNET_i2s(const struct GNUNET_PeerIdentity *pid)
Convert a peer identity to a string (for printing debug messages).
double temperature_init
Initial softmax action-selection temperature.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ GAS_ril_address_preference_feedback()

static void GAS_ril_address_preference_feedback ( void *  solver,
struct GNUNET_SERVICE_Client application,
const struct GNUNET_PeerIdentity peer,
const struct GNUNET_TIME_Relative  scope,
enum GNUNET_ATS_PreferenceKind  kind,
double  score 
)
static

Give feedback about the current assignment.

Parameters
solverthe solver handle
applicationthe application
peerthe peer to change the preference for
scopethe time interval for this feedback: [now - scope .. now]
kindthe kind to change the preference
scorethe score

Definition at line 2491 of file plugin_ats_ril.c.

References GNUNET_ATS_print_preference_type(), GNUNET_ERROR_TYPE_DEBUG, GNUNET_i2s(), LOG, and GNUNET_TIME_Relative::rel_value_us.

Referenced by libgnunet_plugin_ats_ril_init().

2497 {
2499  "API_address_preference_feedback() Peer '%s' got a feedback of %+.3f from application %s for "
2500  "preference %s for %d seconds\n",
2501  GNUNET_i2s (peer),
2502  "UNKNOWN",
2504  scope.rel_value_us / 1000000);
2505 }
uint64_t rel_value_us
The actual value.
const char * GNUNET_ATS_print_preference_type(enum GNUNET_ATS_PreferenceKind type)
Convert a enum GNUNET_ATS_PreferenceType to a string.
#define LOG(kind,...)
const char * GNUNET_i2s(const struct GNUNET_PeerIdentity *pid)
Convert a peer identity to a string (for printing debug messages).
Here is the call graph for this function:
Here is the caller graph for this function:

◆ GAS_ril_bulk_start()

static void GAS_ril_bulk_start ( void *  solver)
static

Start a bulk operation.

Parameters
solverthe solver

Definition at line 2514 of file plugin_ats_ril.c.

References GAS_RIL_Handle::bulk_lock, GNUNET_ERROR_TYPE_DEBUG, and LOG.

Referenced by libgnunet_plugin_ats_ril_init().

2515 {
2516  struct GAS_RIL_Handle *s = solver;
2517 
2519  "API_bulk_start() lock: %d\n", s->bulk_lock + 1);
2520 
2521  s->bulk_lock++;
2522 }
A handle for the reinforcement learning solver.
int bulk_lock
Lock for bulk operations.
#define LOG(kind,...)
Here is the caller graph for this function:

◆ GAS_ril_bulk_stop()

static void GAS_ril_bulk_stop ( void *  solver)
static

Bulk operation done.

Parameters
solverthe solver handle

Definition at line 2531 of file plugin_ats_ril.c.

References GAS_RIL_Handle::bulk_changes, GAS_RIL_Handle::bulk_lock, GNUNET_break, GNUNET_ERROR_TYPE_DEBUG, LOG, and ril_step().

Referenced by libgnunet_plugin_ats_ril_init().

2532 {
2533  struct GAS_RIL_Handle *s = solver;
2534 
2536  "API_bulk_stop() lock: %d\n",
2537  s->bulk_lock - 1);
2538 
2539  if (s->bulk_lock < 1)
2540  {
2541  GNUNET_break (0);
2542  return;
2543  }
2544  s->bulk_lock--;
2545 
2546  if (0 < s->bulk_changes)
2547  {
2548  ril_step (solver);
2549  s->bulk_changes = 0;
2550  }
2551 }
int bulk_changes
Number of changes during a lock.
A handle for the reinforcement learning solver.
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur...
int bulk_lock
Lock for bulk operations.
#define LOG(kind,...)
static void ril_step(struct GAS_RIL_Handle *solver)
Prototype of the ril_step() procedure.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ GAS_ril_get_preferred_address()

static void GAS_ril_get_preferred_address ( void *  solver,
const struct GNUNET_PeerIdentity peer 
)
static

Tell solver to notify ATS if the address to use changes for a specific peer using the bandwidth changed callback.

The solver must only notify about changes for peers with pending address requests!

Parameters
solverthe solver handle
peerthe identity of the peer

Definition at line 2565 of file plugin_ats_ril.c.

References ATS_Address::addr, RIL_Peer_Agent::address_inuse, GNUNET_ATS_PluginEnvironment::bandwidth_changed_cb, RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, GNUNET_ATS_PluginEnvironment::cls, GAS_RIL_Handle::env, envi_set_active_suggestion(), RIL_Learning_Parameters::epsilon, RIL_Learning_Parameters::epsilon_init, GNUNET_ERROR_TYPE_DEBUG, GNUNET_i2s(), GNUNET_YES, RIL_Peer_Agent::is_active, LOG, GAS_RIL_Handle::parameters, ATS_Address::plugin, ril_get_agent(), ril_try_unblock_agent(), RIL_Learning_Parameters::temperature, and RIL_Learning_Parameters::temperature_init.

Referenced by libgnunet_plugin_ats_ril_init().

2567 {
2568  struct GAS_RIL_Handle *s = solver;
2569  struct RIL_Peer_Agent *agent;
2570 
2571  LOG (GNUNET_ERROR_TYPE_DEBUG, "API_get_preferred_address()\n");
2572 
2573  agent = ril_get_agent (s, peer, GNUNET_YES);
2574 
2575  agent->is_active = GNUNET_YES;
2576  envi_set_active_suggestion (solver, agent, agent->address_inuse, agent->bw_in,
2577  agent->bw_out, GNUNET_YES);
2578 
2579  ril_try_unblock_agent (solver, agent, GNUNET_YES);
2580 
2581  if (agent->address_inuse)
2582  {
2584  "API_get_preferred_address() Activated agent for peer '%s' with %s address %s\n",
2585  GNUNET_i2s (peer), agent->address_inuse->plugin,
2586  agent->address_inuse->addr);
2587  }
2588  else
2589  {
2591  "API_get_preferred_address() Activated agent for peer '%s', but no address available\n",
2592  GNUNET_i2s (peer));
2595  }
2596  if (NULL != agent->address_inuse)
2597  s->env->bandwidth_changed_cb (s->env->cls,
2598  agent->address_inuse);
2599 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
int is_active
Whether the agent is active or not.
struct GNUNET_ATS_PluginEnvironment * env
The solver-plugin environment of the solver-plugin API.
A handle for the reinforcement learning solver.
const void * addr
Address (in plugin-specific binary format).
char * plugin
Plugin name.
static void ril_try_unblock_agent(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, int silent)
Unblocks an agent for which a connection request is there, that could not be satisfied.
double epsilon
Ratio, with what probability an agent should explore in the e-greed policy.
static void envi_set_active_suggestion(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, struct ATS_Address *new_address, unsigned long long new_bw_in, unsigned long long new_bw_out, int silent)
Changes the active assignment suggestion of the handler and invokes the bw_changed callback to notify...
double temperature
Softmax action-selection temperature.
struct RIL_Learning_Parameters parameters
Learning parameters.
static struct RIL_Peer_Agent * ril_get_agent(struct GAS_RIL_Handle *solver, const struct GNUNET_PeerIdentity *peer, int create)
Returns the agent for a peer.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
double epsilon_init
Initial exploration ratio value.
void * cls
Closure to pass to all callbacks in this struct.
#define GNUNET_YES
Definition: gnunet_common.h:77
#define LOG(kind,...)
GAS_bandwidth_changed_cb bandwidth_changed_cb
ATS addresses callback to be notified about bandwidth assignment changes.
const char * GNUNET_i2s(const struct GNUNET_PeerIdentity *pid)
Convert a peer identity to a string (for printing debug messages).
double temperature_init
Initial softmax action-selection temperature.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ GAS_ril_stop_get_preferred_address()

static void GAS_ril_stop_get_preferred_address ( void *  solver,
const struct GNUNET_PeerIdentity peer 
)
static

Tell solver stop notifying ATS about changes for this peers.

The solver must only notify about changes for peers with pending address requests!

Parameters
solverthe solver handle
peerthe peer

Definition at line 2612 of file plugin_ats_ril.c.

References RIL_Peer_Agent::address_inuse, RIL_Peer_Agent::bw_in, RIL_Peer_Agent::bw_out, envi_set_active_suggestion(), RIL_Learning_Parameters::epsilon, RIL_Learning_Parameters::epsilon_init, GNUNET_break, GNUNET_ERROR_TYPE_DEBUG, GNUNET_i2s(), GNUNET_NO, GNUNET_YES, RIL_Peer_Agent::is_active, LOG, GAS_RIL_Handle::parameters, ril_get_agent(), ril_step(), RIL_Learning_Parameters::temperature, and RIL_Learning_Parameters::temperature_init.

Referenced by libgnunet_plugin_ats_ril_init().

2614 {
2615  struct GAS_RIL_Handle *s = solver;
2616  struct RIL_Peer_Agent *agent;
2617 
2619  "API_stop_get_preferred_address()");
2620 
2621  agent = ril_get_agent (s, peer, GNUNET_NO);
2622 
2623  if (NULL == agent)
2624  {
2625  GNUNET_break (0);
2626  return;
2627  }
2628  if (GNUNET_NO == agent->is_active)
2629  {
2630  GNUNET_break (0);
2631  return;
2632  }
2633 
2636 
2637  agent->is_active = GNUNET_NO;
2638 
2639  envi_set_active_suggestion (s, agent, agent->address_inuse, agent->bw_in,
2640  agent->bw_out,
2641  GNUNET_YES);
2642 
2643  ril_step (s);
2644 
2646  "API_stop_get_preferred_address() Paused agent for peer '%s'\n",
2647  GNUNET_i2s (peer));
2648 }
uint32_t bw_in
Inbound bandwidth assigned by the agent.
int is_active
Whether the agent is active or not.
A handle for the reinforcement learning solver.
#define GNUNET_NO
Definition: gnunet_common.h:78
double epsilon
Ratio, with what probability an agent should explore in the e-greed policy.
static void envi_set_active_suggestion(struct GAS_RIL_Handle *solver, struct RIL_Peer_Agent *agent, struct ATS_Address *new_address, unsigned long long new_bw_in, unsigned long long new_bw_out, int silent)
Changes the active assignment suggestion of the handler and invokes the bw_changed callback to notify...
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur...
double temperature
Softmax action-selection temperature.
struct RIL_Learning_Parameters parameters
Learning parameters.
static struct RIL_Peer_Agent * ril_get_agent(struct GAS_RIL_Handle *solver, const struct GNUNET_PeerIdentity *peer, int create)
Returns the agent for a peer.
uint32_t bw_out
Outbound bandwidth assigned by the agent.
struct ATS_Address * address_inuse
Address in use.
double epsilon_init
Initial exploration ratio value.
#define GNUNET_YES
Definition: gnunet_common.h:77
#define LOG(kind,...)
static void ril_step(struct GAS_RIL_Handle *solver)
Prototype of the ril_step() procedure.
const char * GNUNET_i2s(const struct GNUNET_PeerIdentity *pid)
Convert a peer identity to a string (for printing debug messages).
double temperature_init
Initial softmax action-selection temperature.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ libgnunet_plugin_ats_ril_init()

void* libgnunet_plugin_ats_ril_init ( void *  cls)

Entry point for the plugin.

Parameters
clspointer to the 'struct GNUNET_ATS_PluginEnvironment'

Definition at line 2657 of file plugin_ats_ril.c.

References _, RIL_Learning_Parameters::algorithm, RIL_Learning_Parameters::alpha, GNUNET_ATS_PluginEnvironment::bandwidth_changed_cb, RIL_Learning_Parameters::beta, RIL_Scope::bw_in_available, RIL_Scope::bw_out_available, GNUNET_ATS_PluginEnvironment::cfg, GNUNET_ATS_SolverFunctions::cls, GNUNET_ATS_PluginEnvironment::cls, GAS_RIL_Handle::done, RIL_Learning_Parameters::eligibility_trace_mode, env, GAS_RIL_Handle::env, RIL_Learning_Parameters::epsilon, RIL_Learning_Parameters::epsilon_decay, RIL_Learning_Parameters::epsilon_init, RIL_Learning_Parameters::gamma, GAS_ril_address_add(), GAS_ril_address_change_preference(), GAS_ril_address_delete(), GAS_ril_address_preference_feedback(), GAS_ril_address_property_changed(), GAS_ril_bulk_start(), GAS_ril_bulk_stop(), GAS_ril_get_preferred_address(), GAS_ril_stop_get_preferred_address(), GNUNET_ATS_PluginEnvironment::get_preferences, GNUNET_assert, GNUNET_CONFIGURATION_get_value_float(), GNUNET_CONFIGURATION_get_value_number(), GNUNET_CONFIGURATION_get_value_string(), GNUNET_CONFIGURATION_get_value_time(), GNUNET_CONFIGURATION_get_value_yesno(), GNUNET_ERROR_TYPE_DEBUG, GNUNET_ERROR_TYPE_ERROR, GNUNET_ERROR_TYPE_INFO, GNUNET_free, GNUNET_malloc, GNUNET_new, GNUNET_NO, GNUNET_NT_to_string(), GNUNET_OK, GNUNET_STRINGS_utf8_toupper(), GNUNET_SYSERR, GNUNET_YES, GNUNET_ATS_PluginEnvironment::in_quota, RIL_Learning_Parameters::lambda, LOG, GNUNET_ATS_PluginEnvironment::network_count, GAS_RIL_Handle::network_entries, GAS_RIL_Handle::networks_count, GNUNET_ATS_PluginEnvironment::out_quota, GAS_RIL_Handle::parameters, RIL_Learning_Parameters::rbf_divisor, RIL_ALGO_Q, RIL_ALGO_SARSA, RIL_DEFAULT_ALGORITHM, RIL_DEFAULT_DISCOUNT_BETA, RIL_DEFAULT_DISCOUNT_GAMMA, RIL_DEFAULT_EXPLORE_DECAY, RIL_DEFAULT_EXPLORE_RATIO, RIL_DEFAULT_GRADIENT_STEP_SIZE, RIL_DEFAULT_RBF_DIVISOR, RIL_DEFAULT_SELECT, RIL_DEFAULT_STEP_TIME_MAX, RIL_DEFAULT_STEP_TIME_MIN, RIL_DEFAULT_TEMPERATURE, RIL_DEFAULT_TEMPERATURE_DECAY, RIL_DEFAULT_TRACE_DECAY, RIL_DEFAULT_WELFARE, RIL_E_ACCUMULATE, RIL_E_REPLACE, RIL_SELECT_EGREEDY, RIL_SELECT_SOFTMAX, RIL_WELFARE_EGALITARIAN, RIL_WELFARE_NASH, GNUNET_ATS_SolverFunctions::s_add, GNUNET_ATS_SolverFunctions::s_address_update_property, GNUNET_ATS_SolverFunctions::s_bulk_start, GNUNET_ATS_SolverFunctions::s_bulk_stop, GNUNET_ATS_SolverFunctions::s_del, GNUNET_ATS_SolverFunctions::s_feedback, GNUNET_ATS_SolverFunctions::s_get, GNUNET_ATS_SolverFunctions::s_get_stop, GNUNET_ATS_SolverFunctions::s_pref, RIL_Learning_Parameters::select, sf, GAS_RIL_Handle::simulate, RIL_Learning_Parameters::social_welfare, GNUNET_ATS_PluginEnvironment::stats, GAS_RIL_Handle::step_count, RIL_Learning_Parameters::step_time_max, RIL_Learning_Parameters::step_time_min, RIL_Learning_Parameters::temperature, RIL_Learning_Parameters::temperature_decay, RIL_Learning_Parameters::temperature_init, and RIL_Scope::type.

2658 {
2659  static struct GNUNET_ATS_SolverFunctions sf;
2661  struct GAS_RIL_Handle *solver = GNUNET_new (struct GAS_RIL_Handle);
2662  struct RIL_Scope *cur;
2663  int c;
2664  char *string;
2665  float f_tmp;
2666 
2668  "API_init() Initializing RIL solver\n");
2669 
2670  GNUNET_assert (NULL != env);
2671  GNUNET_assert (NULL != env->cfg);
2672  GNUNET_assert (NULL != env->stats);
2673  GNUNET_assert (NULL != env->bandwidth_changed_cb);
2674  GNUNET_assert (NULL != env->get_preferences);
2675 
2677  "RIL_RBF_DIVISOR",
2678  &solver->parameters.
2679  rbf_divisor))
2680  {
2682  }
2683 
2684  if (GNUNET_OK
2685  != GNUNET_CONFIGURATION_get_value_time (env->cfg, "ats",
2686  "RIL_STEP_TIME_MIN",
2687  &solver->parameters.step_time_min))
2688  {
2690  }
2691 
2692  if (GNUNET_OK
2693  != GNUNET_CONFIGURATION_get_value_time (env->cfg, "ats",
2694  "RIL_STEP_TIME_MAX",
2695  &solver->parameters.step_time_max))
2696  {
2698  }
2699 
2701  "RIL_ALGORITHM",
2702  &string))
2703  {
2704  GNUNET_STRINGS_utf8_toupper (string, string);
2705  if (0 == strcmp (string, "SARSA"))
2706  {
2708  }
2709  if (0 == strcmp (string, "Q-LEARNING"))
2710  {
2711  solver->parameters.algorithm = RIL_ALGO_Q;
2712  }
2713 
2714  GNUNET_free (string);
2715  }
2716  else
2717  {
2719  }
2720 
2722  "RIL_SELECT",
2723  &string))
2724  {
2725  solver->parameters.select = ! strcmp (string, "EGREEDY") ?
2727  GNUNET_free (string);
2728  }
2729  else
2730  {
2732  }
2733 
2734 
2737  "RIL_DISCOUNT_BETA",
2738  &f_tmp))
2739  {
2740  if (f_tmp < 0.0)
2741  {
2742  LOG (GNUNET_ERROR_TYPE_ERROR, _ ("Invalid %s configuration %f \n"),
2743  "RIL_DISCOUNT_BETA", f_tmp);
2744  }
2745  else
2746  {
2747  solver->parameters.beta = f_tmp;
2748  LOG (GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n",
2749  "RIL_DISCOUNT_BETA", f_tmp);
2750  }
2751  }
2752 
2755  "RIL_DISCOUNT_GAMMA",
2756  &f_tmp))
2757  {
2758  if ((f_tmp < 0.0) || (f_tmp > 1.0))
2759  {
2760  LOG (GNUNET_ERROR_TYPE_ERROR, _ ("Invalid %s configuration %f \n"),
2761  "RIL_DISCOUNT_GAMMA", f_tmp);
2762  }
2763  else
2764  {
2765  solver->parameters.gamma = f_tmp;
2766  LOG (GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n",
2767  "RIL_DISCOUNT_GAMMA", f_tmp);
2768  }
2769  }
2770 
2773  "RIL_GRADIENT_STEP_SIZE",
2774  &f_tmp))
2775  {
2776  if ((f_tmp < 0.0) || (f_tmp > 1.0))
2777  {
2778  LOG (GNUNET_ERROR_TYPE_ERROR, _ ("Invalid %s configuration %f \n"),
2779  "RIL_GRADIENT_STEP_SIZE", f_tmp);
2780  }
2781  else
2782  {
2783  solver->parameters.alpha = f_tmp;
2784  LOG (GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n",
2785  "RIL_GRADIENT_STEP_SIZE", f_tmp);
2786  }
2787  }
2788 
2791  "RIL_TRACE_DECAY",
2792  &f_tmp))
2793  {
2794  if ((f_tmp < 0.0) || (f_tmp > 1.0))
2795  {
2796  LOG (GNUNET_ERROR_TYPE_ERROR, _ ("Invalid %s configuration %f \n"),
2797  "RIL_TRACE_DECAY", f_tmp);
2798  }
2799  else
2800  {
2801  solver->parameters.lambda = f_tmp;
2802  LOG (GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n",
2803  "RIL_TRACE_DECAY", f_tmp);
2804  }
2805  }
2806 
2809  "RIL_EXPLORE_RATIO",
2810  &f_tmp))
2811  {
2812  if ((f_tmp < 0.0) || (f_tmp > 1.0))
2813  {
2814  LOG (GNUNET_ERROR_TYPE_ERROR, _ ("Invalid %s configuration %f \n"),
2815  "RIL_EXPLORE_RATIO", f_tmp);
2816  }
2817  else
2818  {
2819  solver->parameters.epsilon_init = f_tmp;
2820  LOG (GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n",
2821  "RIL_EXPLORE_RATIO", f_tmp);
2822  }
2823  }
2824 
2827  "RIL_EXPLORE_DECAY",
2828  &f_tmp))
2829  {
2830  if ((f_tmp < 0.0) || (f_tmp > 1.0))
2831  {
2832  LOG (GNUNET_ERROR_TYPE_ERROR, _ ("Invalid %s configuration %f \n"),
2833  "RIL_EXPLORE_DECAY", f_tmp);
2834  }
2835  else
2836  {
2837  solver->parameters.epsilon_decay = f_tmp;
2838  LOG (GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n",
2839  "RIL_EXPLORE_DECAY", f_tmp);
2840  }
2841  }
2842 
2845  "RIL_TEMPERATURE",
2846  &f_tmp))
2847  {
2848  if (f_tmp <= 0.0)
2849  {
2850  LOG (GNUNET_ERROR_TYPE_ERROR, _ ("Invalid %s configuration %f \n"),
2851  "RIL_TEMPERATURE", f_tmp);
2852  }
2853  else
2854  {
2855  solver->parameters.temperature_init = f_tmp;
2856  LOG (GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n",
2857  "RIL_TEMPERATURE", f_tmp);
2858  }
2859  }
2860 
2863  "RIL_TEMPERATURE_DECAY",
2864  &f_tmp))
2865  {
2866  if ((f_tmp <= 0.0) ||(solver->parameters.temperature_decay > 1) )
2867  {
2868  LOG (GNUNET_ERROR_TYPE_ERROR, _ ("Invalid %s configuration %f \n"),
2869  "RIL_TEMPERATURE_DECAY", f_tmp);
2870  }
2871  else
2872  {
2873  solver->parameters.temperature_decay = f_tmp;
2874  LOG (GNUNET_ERROR_TYPE_INFO, "Using %s of %.3f\n",
2875  "RIL_TEMPERATURE_DECAY", f_tmp);
2876  }
2877  }
2878 
2880  "RIL_SIMULATE",
2881  &solver->simulate))
2882  {
2883  solver->simulate = GNUNET_NO;
2884  }
2885 
2887  "RIL_REPLACE_TRACES"))
2888  {
2890  }
2891  else
2892  {
2894  }
2895 
2897  "RIL_SOCIAL_WELFARE",
2898  &string))
2899  {
2900  solver->parameters.social_welfare = ! strcmp (string, "NASH") ?
2903  GNUNET_free (string);
2904  }
2905  else
2906  {
2908  }
2909 
2910  solver->env = env;
2911  sf.cls = solver;
2921 
2922  solver->networks_count = env->network_count;
2923  solver->network_entries = GNUNET_malloc (env->network_count * sizeof(struct
2924  RIL_Scope));
2925  solver->step_count = 0;
2926  solver->done = GNUNET_NO;
2927 
2928  for (c = 0; c < env->network_count; c++)
2929  {
2930  cur = &solver->network_entries[c];
2931  cur->type = c;
2932  cur->bw_in_available = env->in_quota[c];
2933  cur->bw_out_available = env->out_quota[c];
2935  "init() Quotas for %s network: IN %llu - OUT %llu\n",
2936  GNUNET_NT_to_string (cur->type),
2937  cur->bw_in_available / 1024,
2938  cur->bw_out_available / 1024);
2939  }
2940 
2941  LOG (GNUNET_ERROR_TYPE_DEBUG, "init() Parameters:\n");
2943  "init() Algorithm = %s, alpha = %f, beta = %f, lambda = %f\n",
2944  solver->parameters.algorithm ? "Q" : "SARSA",
2945  solver->parameters.alpha,
2946  solver->parameters.beta,
2947  solver->parameters.lambda);
2949  "init() exploration_ratio = %f, temperature = %f, ActionSelection = %s\n",
2950  solver->parameters.epsilon,
2951  solver->parameters.temperature,
2952  solver->parameters.select ? "EGREEDY" : "SOFTMAX");
2953  LOG (GNUNET_ERROR_TYPE_DEBUG, "init() RBF_DIVISOR = %llu\n",
2954  solver->parameters.rbf_divisor);
2955 
2956  return &sf;
2957 }
GAS_solver_stop_get_preferred_address s_get_stop
Tell solver stop notifying ATS about changes for this peers.
static void GAS_ril_address_change_preference(void *solver, const struct GNUNET_PeerIdentity *peer, enum GNUNET_ATS_PreferenceKind kind, double pref_rel)
Change relative preference for quality in solver.
static void GAS_ril_address_delete(void *solver, struct ATS_Address *address)
Delete an address in the solver.
struct GNUNET_ATS_PluginEnvironment * env
The solver-plugin environment of the solver-plugin API.
int GNUNET_CONFIGURATION_get_value_time(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, struct GNUNET_TIME_Relative *time)
Get a configuration value that should be a relative time.
GAS_solver_address_add s_add
Add a new address for a peer to the solver.
int GNUNET_CONFIGURATION_get_value_number(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, unsigned long long *number)
Get a configuration value that should be a number.
const char * GNUNET_NT_to_string(enum GNUNET_NetworkType net)
Convert a enum GNUNET_NetworkType to a string.
Definition: nt.c:44
#define RIL_DEFAULT_SELECT
double lambda
Trace-decay factor for eligibility traces.
struct GNUNET_STATISTICS_Handle * stats
Statistics handle to be used by the solver.
#define RIL_DEFAULT_DISCOUNT_GAMMA
#define RIL_DEFAULT_EXPLORE_DECAY
#define RIL_DEFAULT_RBF_DIVISOR
static void GAS_ril_address_add(void *solver, struct ATS_Address *address, uint32_t network)
Add a new address for a peer to the solver.
GAS_get_preferences get_preferences
ATS addresses function to obtain preference values.
static void GAS_ril_bulk_stop(void *solver)
Bulk operation done.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
static void GAS_ril_stop_get_preferred_address(void *solver, const struct GNUNET_PeerIdentity *peer)
Tell solver stop notifying ATS about changes for this peers.
A handle for the reinforcement learning solver.
double gamma
Learning discount factor in the TD-update for MDPs.
struct GNUNET_TIME_Relative step_time_max
Maximum interval time between steps in milliseconds.
#define RIL_DEFAULT_GRADIENT_STEP_SIZE
GAS_solver_address_feedback_preference s_feedback
Give feedback about the current assignment.
int GNUNET_CONFIGURATION_get_value_float(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, float *number)
Get a configuration value that should be a floating point number.
GAS_solver_address_property_changed s_address_update_property
Update the properties of an address in the solver.
double temperature_decay
Decay factor of the temperature value.
#define RIL_DEFAULT_WELFARE
#define GNUNET_NO
Definition: gnunet_common.h:78
struct RIL_Scope * network_entries
Array of networks with global assignment state.
#define GNUNET_OK
Named constants for return values.
Definition: gnunet_common.h:75
#define GNUNET_new(type)
Allocate a struct or union of the given type.
double epsilon
Ratio, with what probability an agent should explore in the e-greed policy.
static void GAS_ril_address_property_changed(void *solver, struct ATS_Address *address)
Update the properties of an address in the solver.
unsigned long long step_count
Number of performed steps.
double temperature
Softmax action-selection temperature.
#define _(String)
GNU gettext support macro.
Definition: platform.h:181
#define RIL_DEFAULT_TRACE_DECAY
unsigned long long simulate
Simulate steps, i.e.
void GNUNET_STRINGS_utf8_toupper(const char *input, char *output)
Convert the utf-8 input string to upper case.
Definition: strings.c:578
static void GAS_ril_get_preferred_address(void *solver, const struct GNUNET_PeerIdentity *peer)
Tell solver to notify ATS if the address to use changes for a specific peer using the bandwidth chang...
#define RIL_DEFAULT_ALGORITHM
enum RIL_Select select
Action selection strategy;.
enum GNUNET_NetworkType type
ATS network type.
enum RIL_Welfare social_welfare
Which measure of social welfare should be used.