{ localUrl: '../page/avert_self_improvement.html', arbitalUrl: 'https://arbital.com/p/avert_self_improvement', rawJsonUrl: '../raw/2x3.json', likeableId: '1827', likeableType: 'page', myLikeValue: '0', likeCount: '1', dislikeCount: '0', likeScore: '1', individualLikes: [ 'EricBruylant' ], pageId: 'avert_self_improvement', edit: '1', editSummary: '', prevEdit: '0', currentEdit: '1', wasPublished: 'true', type: 'wiki', title: 'Averting the convergent instrumental strategy of self-improvement', clickbait: 'We probably want the first AGI to *not* improve as fast as possible, but improving as fast as possible is a convergent strategy for accomplishing most things.', textLength: '1080', alias: 'avert_self_improvement', externalUrl: '', sortChildrenBy: 'likes', hasVote: 'false', voteType: '', votesAnonymous: 'false', editCreatorId: 'EliezerYudkowsky', editCreatedAt: '2016-03-28 01:51:31', pageCreatorId: 'EliezerYudkowsky', pageCreatedAt: '2016-03-28 01:51:31', seeDomainId: '0', editDomainId: 'EliezerYudkowsky', submitToDomainId: '0', isAutosave: 'false', isSnapshot: 'false', isLiveEdit: 'true', isMinorEdit: 'false', indirectTeacher: 'false', todoCount: '0', isEditorComment: 'false', isApprovedComment: 'true', isResolved: 'false', snapshotText: '', anchorContext: '', anchorText: '', anchorOffset: '0', mergedInto: '', isDeleted: 'false', viewCount: '93', text: 'Rapid capability gains, or just large capability gains between a training paradigm and a test paradigm, are one of the primary expected reasons why AGI alignment might be hard. We probably want the first AGI or AGIs ever built, tested, and used to *not* self-improve as quickly as possible. Since there's a very strong [10g convergent incentive] to self-improve and do things [42 neighboring] to self-improvement, by default you would expect an AGI to search for ways to defeat naive blocks on self-improvement, which violates the [2x4 nonadversarial principle]. Thus, any proposals to limit an AGI's capabilities imply a very strong desideratum for us to figure out a way to [2vk avert the instrumental incentive] to self-improvement in that AGI. The alternative is failing the [2x Omni Test], violating the nonadversarial principle, [2rb having the AGI's code be actively inconsistent with what the AGI would approve of its own code being] (if the brake is a code-level measure), and setting up a safety measure that the AGI wants to defeat as the [2x4 only line of defense].', metaText: '', isTextLoaded: 'true', isSubscribedToDiscussion: 'false', isSubscribedToUser: 'false', isSubscribedAsMaintainer: 'false', discussionSubscriberCount: '1', maintainerCount: '1', userSubscriberCount: '0', lastVisit: '', hasDraft: 'false', votes: [], voteSummary: 'null', muVoteSummary: '0', voteScaling: '0', currentUserVote: '-2', voteCount: '0', lockedVoteType: '', maxEditEver: '0', redLinkCount: '0', lockedBy: '', lockedUntil: '', nextPageId: '', prevPageId: '', usedAsMastery: 'false', proposalEditNum: '0', permissions: { edit: { has: 'false', reason: 'You don't have domain permission to edit this page' }, proposeEdit: { has: 'true', reason: '' }, delete: { has: 'false', reason: 'You don't have domain permission to delete this page' }, comment: { has: 'false', reason: 'You can't comment in this domain because you are not a member' }, proposeComment: { has: 'true', reason: '' } }, summaries: {}, creatorIds: [ 'EliezerYudkowsky' ], childIds: [], parentIds: [ 'corrigibility' ], commentIds: [], questionIds: [], tagIds: [ 'value_alignment_open_problem', 'stub_meta_tag' ], relatedIds: [], markIds: [], explanations: [], learnMore: [], requirements: [], subjects: [], lenses: [], lensParentId: '', pathPages: [], learnMoreTaughtMap: {}, learnMoreCoveredMap: {}, learnMoreRequiredMap: {}, editHistory: {}, domainSubmissions: {}, answers: [], answerCount: '0', commentCount: '0', newCommentCount: '0', linkedMarkCount: '0', changeLogs: [ { likeableId: '0', likeableType: 'changeLog', myLikeValue: '0', likeCount: '0', dislikeCount: '0', likeScore: '0', individualLikes: [], id: '9147', pageId: 'avert_self_improvement', userId: 'EliezerYudkowsky', edit: '1', type: 'newEdit', createdAt: '2016-03-28 01:51:31', auxPageId: '', oldSettingsValue: '', newSettingsValue: '' }, { likeableId: '0', likeableType: 'changeLog', myLikeValue: '0', likeCount: '0', dislikeCount: '0', likeScore: '0', individualLikes: [], id: '9128', pageId: 'avert_self_improvement', userId: 'EliezerYudkowsky', edit: '0', type: 'deleteTag', createdAt: '2016-03-27 19:59:54', auxPageId: 'stub_meta_tag', oldSettingsValue: '', newSettingsValue: '' } ], feedSubmissions: [], searchStrings: {}, hasChildren: 'false', hasParents: 'true', redAliases: {}, improvementTagIds: [], nonMetaTagIds: [], todos: [], slowDownMap: 'null', speedUpMap: 'null', arcPageIds: 'null', contentRequests: {} }